From c3b490f8b206f7be33bcb72b98bc0b25ca93b706 Mon Sep 17 00:00:00 2001 From: Younes ENNAJI Date: Fri, 23 Oct 2026 09:40:15 +0000 Subject: [PATCH] Walk the parser's own tree The translation layer between the parser's AST and the shape walkers is gone: the walkers read the nodes the parser built, the alternations come without their separator tokens and the empty branches without their patch-up passes, and the quantifier bounds come from the parser's own reading instead of a re-parse of the token text. IgnoredRegexValidator reads the same tree, its branch text sliced from the source between node positions so escapes come back as they were written. A padded hex escape still types 'ax{ 41 }' for PHP 7.4 and the letter A on 8.x, the inline (?i:...) groups still drop the constants their caseless body cannot promise, and every resolver fixture says the same thing it said through the translation: 1873 of 1873. --- src/Command/IgnoredRegexValidator.php | 67 +- .../ValidateIgnoredErrorsExtension.php | 4 +- src/Type/Regex/PHPRegexReader.php | 64 ++ src/Type/Regex/PHPRegexTreeNodeBuilder.php | 514 ---------------- src/Type/Regex/RegexGroupParser.php | 574 +++++++++--------- src/Type/Regex/RegexTreeNode.php | 78 --- .../Command/IgnoredRegexValidatorTest.php | 4 +- .../Type/Regex/RegexShapeReadingTest.php | 89 +-- 8 files changed, 445 insertions(+), 949 deletions(-) create mode 100644 src/Type/Regex/PHPRegexReader.php delete mode 100644 src/Type/Regex/PHPRegexTreeNodeBuilder.php delete mode 100644 src/Type/Regex/RegexTreeNode.php diff --git a/src/Command/IgnoredRegexValidator.php b/src/Command/IgnoredRegexValidator.php index 2517b8cfd72..e504917dd29 100644 --- a/src/Command/IgnoredRegexValidator.php +++ b/src/Command/IgnoredRegexValidator.php @@ -4,12 +4,15 @@ use Nette\Utils\Strings; use PHPRegex\Parser\Exception\RegexException; +use PHPRegex\Parser\Node\AlternationNode; +use PHPRegex\Parser\Node\AnchorNode; +use PHPRegex\Parser\Node\NodeInterface; +use PHPRegex\Parser\Node\SequenceNode; use PHPStan\PhpDoc\TypeStringResolver; use PHPStan\PhpDocParser\Parser\ParserException; use PHPStan\ShouldNotHappenException; use PHPStan\Type\ObjectType; -use PHPStan\Type\Regex\PHPRegexTreeNodeBuilder; -use PHPStan\Type\Regex\RegexTreeNode; +use PHPStan\Type\Regex\PHPRegexReader; use PHPStan\Type\VerbosityLevel; use function count; use function strrpos; @@ -20,7 +23,7 @@ final class IgnoredRegexValidator public function __construct( private TypeStringResolver $typeStringResolver, - private PHPRegexTreeNodeBuilder $treeNodeBuilder, + private PHPRegexReader $reader, ) { } @@ -31,7 +34,7 @@ public function validate(string $regex): IgnoredRegexValidatorResult $regex = $this->removeDelimiters($regex); try { - $ast = $this->treeNodeBuilder->parse($regex, '', $delimiter); + $root = $this->reader->read($regex, '', $delimiter); } catch (RegexException) { return new IgnoredRegexValidatorResult(ignoredTypes: [], anchorsInTheMiddle: false, allErrorsIgnored: false); } @@ -44,8 +47,8 @@ public function validate(string $regex): IgnoredRegexValidatorResult } return new IgnoredRegexValidatorResult( - ignoredTypes: $this->getIgnoredTypes($ast), - anchorsInTheMiddle: $this->hasAnchorsInTheMiddle($ast), + ignoredTypes: $this->getIgnoredTypes($root->pattern, $root->source ?? $regex), + anchorsInTheMiddle: $this->hasAnchorsInTheMiddle($root->pattern), allErrorsIgnored: false, ); } @@ -53,20 +56,15 @@ public function validate(string $regex): IgnoredRegexValidatorResult /** * @return array */ - private function getIgnoredTypes(RegexTreeNode $ast): array + private function getIgnoredTypes(NodeInterface $ast, string $source): array { - $alternation = $ast->getChild(0); - if ($alternation === null) { - return []; - } - - if ($alternation->getId() !== '#alternation') { + if (!$ast instanceof AlternationNode) { return []; } $types = []; - foreach ($alternation->getChildren() as $child) { - $text = $this->getText($child); + foreach ($ast->alternatives as $alternative) { + $text = $this->getText($alternative, $source); if ($text === null) { continue; } @@ -109,16 +107,17 @@ private function removeDelimiters(string $regex): string return substr($regex, 1, $endDelimiterPosition - 1); } - private function getText(RegexTreeNode $treeNode): ?string + /** + * The text of a branch as it was written, escapes included: the tree + * decodes them, so each leaf is sliced from the source between its + * positions, whatever the leaf spells. + */ + private function getText(NodeInterface $node, string $source): ?string { - if ($treeNode->getId() === 'token') { - return $treeNode->getValueValue(); - } - - if ($treeNode->getId() === '#concatenation') { + if ($node instanceof SequenceNode) { $fullText = ''; - foreach ($treeNode->getChildren() as $child) { - $text = $this->getText($child); + foreach ($node->children as $child) { + $text = $this->getText($child, $source); if ($text === null) { continue; } @@ -133,22 +132,26 @@ private function getText(RegexTreeNode $treeNode): ?string return $fullText; } - return null; + if ($node->getChildren() !== []) { + return null; + } + + return substr($source, $node->getStartPosition(), $node->getEndPosition() - $node->getStartPosition()); } - private function hasAnchorsInTheMiddle(RegexTreeNode $ast): bool + private function hasAnchorsInTheMiddle(NodeInterface $ast): bool { - if ($ast->getId() === 'token') { - $valueArray = $ast->getValue(); - - return $valueArray['token'] === 'anchor' && $valueArray['value'] === '$'; + if ($ast instanceof AnchorNode) { + return $ast->value === '$'; } - $childrenCount = count($ast->getChildren()); - foreach ($ast->getChildren() as $i => $child) { + + $children = $ast instanceof SequenceNode ? $ast->children : $ast->getChildren(); + $childrenCount = count($children); + foreach ($children as $i => $child) { $has = $this->hasAnchorsInTheMiddle($child); if ( $has - && ($ast->getId() !== '#concatenation' || $i !== $childrenCount - 1) + && (!$ast instanceof SequenceNode || $i !== $childrenCount - 1) ) { return true; } diff --git a/src/DependencyInjection/ValidateIgnoredErrorsExtension.php b/src/DependencyInjection/ValidateIgnoredErrorsExtension.php index 51bfe7807ab..ab773c42841 100644 --- a/src/DependencyInjection/ValidateIgnoredErrorsExtension.php +++ b/src/DependencyInjection/ValidateIgnoredErrorsExtension.php @@ -10,7 +10,7 @@ use PHPStan\Analyser\NameScope; use PHPStan\Command\IgnoredRegexValidator; use PHPStan\File\FileExcluder; -use PHPStan\Type\Regex\PHPRegexTreeNodeBuilder; +use PHPStan\Type\Regex\PHPRegexReader; use PHPStan\Php\ComposerPhpVersionFactory; use PHPStan\Php\ConfiguredPhpVersionRangeHelper; use PHPStan\Php\PcreVersion; @@ -129,7 +129,7 @@ public function resolveTypeAlias(string $aliasName, NameScope $nameScope): ?Type // ignoreErrors patterns are matched by the running PHPStan // itself, so their read is judged for the running PHP and // its engine - new PHPRegexTreeNodeBuilder( + new PHPRegexReader( new PhpVersion(PHP_VERSION_ID), PcreVersion::fromRuntime(), ), diff --git a/src/Type/Regex/PHPRegexReader.php b/src/Type/Regex/PHPRegexReader.php new file mode 100644 index 00000000000..776411eaa02 --- /dev/null +++ b/src/Type/Regex/PHPRegexReader.php @@ -0,0 +1,64 @@ +parser ??= RegexParser::create($this->targetOptions()); + + return $this->parser->parsePattern($rawRegex, $modifiers, $delimiter); + } + + /** + * @return array + */ + private function targetOptions(): array + { + $phpVersionId = $this->phpVersion->getVersionId(); + $bundled = PcreTarget::bundledWith($phpVersionId); + + $options = ['php_version' => $phpVersionId]; + if (!$this->pcreVersion->atLeast($bundled->pcreVersion)) { + $options['pcre_version'] = $this->pcreVersion->getVersionString(); + } + + return $options; + } + +} diff --git a/src/Type/Regex/PHPRegexTreeNodeBuilder.php b/src/Type/Regex/PHPRegexTreeNodeBuilder.php deleted file mode 100644 index 806b7b7dfdc..00000000000 --- a/src/Type/Regex/PHPRegexTreeNodeBuilder.php +++ /dev/null @@ -1,514 +0,0 @@ -parser ??= RegexParser::create($this->targetOptions()); - - $ast = $this->parser->parsePattern($rawRegex, $modifiers, $delimiter); - - $body = self::convertBody($ast->pattern); - - return new RegexTreeNode('#expression', null, $body === null ? [] : [$body]); - } - - /** - * @return array - */ - private function targetOptions(): array - { - $phpVersionId = $this->phpVersion->getVersionId(); - $bundled = PcreTarget::bundledWith($phpVersionId); - - $options = ['php_version' => $phpVersionId]; - if (!$this->pcreVersion->atLeast($bundled->pcreVersion)) { - $options['pcre_version'] = $this->pcreVersion->getVersionString(); - } - - return $options; - } - - /** - * A body position (root, group body, quantified atom): a sequence of one - * unit collapses to that unit, an empty body disappears, several units or - * literal characters are wrapped in a concatenation, as the grammar does. - */ - private static function convertBody(NodeInterface $node): ?RegexTreeNode - { - if ($node instanceof SequenceNode) { - return self::fold($node->children); - } - - if ($node instanceof LiteralNode && $node->value === '') { - return null; - } - - $tokens = self::convert($node); - - return self::wrap($tokens); - } - - /** - * @param list $nodes - */ - private static function fold(array $nodes): ?RegexTreeNode - { - $tokens = []; - foreach ($nodes as $child) { - if ($child instanceof LiteralNode && $child->value === '') { - continue; - } - - $tokens = array_merge($tokens, self::convert($child)); - } - - return self::wrap($tokens); - } - - /** - * @param list $tokens - */ - private static function wrap(array $tokens): ?RegexTreeNode - { - if ($tokens === []) { - return null; - } - - if (count($tokens) === 1) { - return $tokens[0]; - } - - return new RegexTreeNode('#concatenation', null, $tokens); - } - - /** - * @return list - */ - private static function convert(NodeInterface $node): array - { - if ($node instanceof LiteralNode) { - $tokens = []; - // one token per character, as the grammar emits; a literal "." is - // always escaped here so it stays a constant instead of any-char - foreach (mb_str_split($node->value, encoding: 'UTF-8') as $char) { - $tokens[] = self::token('literal', $char === '.' ? '\\.' : $char); - } - - return $tokens; - } - - if ($node instanceof DotNode) { - return [self::token('literal', '.')]; - } - - if ($node instanceof AnchorNode) { - return [self::token('anchor', $node->value)]; - } - - if ($node instanceof AssertionNode) { - return [self::token('anchor', '\\' . $node->value)]; - } - - if ($node instanceof KeepNode) { - return [self::token('match_point_reset', '\\K')]; - } - - if ($node instanceof CharTypeNode) { - return [self::token('character_type', '\\' . $node->value)]; - } - - if ($node instanceof UnicodePropNode) { - return [self::token('character_type', self::unicodeProperty($node))]; - } - - if ($node instanceof CharLiteralNode) { - return [self::token('dynamic_character', self::character($node->codePoint))]; - } - - if ($node instanceof ControlCharNode) { - return [self::token('character', self::character($node->codePoint))]; - } - - // the 2017 grammar reads these as literal characters: "1" for \1, - // "k" for \k, "Q...E" for \Q...\E. An unknown token name instead - // makes the walkers drop their constant literals and fall back to - // string types, which is what a backreference actually guarantees. - if ($node instanceof BackrefNode) { - return [self::token('backreference', $node->ref)]; - } - - if ($node instanceof SubroutineNode) { - return [self::token('subroutine', $node->reference)]; - } - - if ($node instanceof CalloutNode) { - return [self::token('callout', '')]; - } - - if ($node instanceof PcreVerbNode) { - if (str_starts_with($node->verb, 'MARK:')) { - return [new RegexTreeNode('#mark', null, [self::token('name', substr($node->verb, 5))])]; - } - - return [self::token('pcre_verb', $node->verb)]; - } - - if ($node instanceof CommentNode) { - return [new RegexTreeNode('#comment', null, [self::token('comment', $node->comment)])]; - } - - if ($node instanceof GroupNode) { - return [self::convertGroup($node)]; - } - - if ($node instanceof QuantifierNode) { - return [self::convertQuantifier($node)]; - } - - if ($node instanceof CharClassNode) { - return [self::convertClass($node)]; - } - - if ($node instanceof ConditionalNode) { - return [self::convertConditional($node)]; - } - - if ($node instanceof DefineNode) { - return self::convertDefinition($node); - } - - if ($node instanceof AlternationNode) { - return [self::convertAlternation($node)]; - } - - if ($node instanceof SequenceNode) { - return array_merge(...array_map(self::convert(...), $node->children)); - } - - return [self::token('phpregex_node', $node::class)]; - } - - private static function convertAlternation(AlternationNode $node): RegexTreeNode - { - $children = []; - foreach ($node->alternatives as $i => $alternative) { - if ($i > 0) { - $children[] = self::token('alternation', '|'); - } - - $body = self::convertBody($alternative); - if ($body !== null) { - $children[] = $body; - } - } - - return new RegexTreeNode('#alternation', null, $children); - } - - private static function convertGroup(GroupNode $node): RegexTreeNode - { - $body = self::convertBody($node->child); - - if ($node->type === GroupType::Named()) { - $children = [self::token('capturing_name', (string) $node->name)]; - if ($body !== null) { - $children[] = $body; - } - - return new RegexTreeNode('#namedcapturing', null, $children); - } - - $id = match ($node->type) { - GroupType::Capturing() => '#capturing', - GroupType::NonCapturing() => '#noncapturing', - GroupType::Atomic() => '#atomicgroup', - GroupType::BranchReset() => '#noncapturingreset', - GroupType::LookaheadPositive() => '#lookahead', - GroupType::LookaheadNegative() => '#negativelookahead', - GroupType::LookbehindPositive() => '#lookbehind', - GroupType::LookbehindNegative() => '#negativelookbehind', - GroupType::InlineFlags(), GroupType::ScanSubstring() => '#noncapturing', - }; - - if ($node->type === GroupType::InlineFlags() && $body === null) { - // an option-setting group like (?i) stands on its own - return new RegexTreeNode('#internal_options', null, [ - self::token('internal_option', '(?' . $node->flags . ')'), - ]); - } - - $children = []; - if ($node->type === GroupType::InlineFlags()) { - $children[] = self::token('non_capturing_internal_option', '(?' . $node->flags . ':'); - } - - if ($body !== null) { - $children[] = $body; - } - - return new RegexTreeNode($id, null, $children); - } - - private static function convertQuantifier(QuantifierNode $node): RegexTreeNode - { - $bounds = QuantifierBounds::parse($node->quantifier); - - $suffix = match ($node->type) { - QuantifierType::Lazy() => '_lazy', - QuantifierType::Possessive() => '_possessive', - QuantifierType::Greedy() => '', - }; - - if ($bounds === null) { - return new RegexTreeNode('#quantification', null, array_merge(self::convert($node->node), [ - self::token('quantifier' . $suffix, $node->quantifier), - ])); - } - - $min = $bounds->min; - $max = $bounds->max; - - if ($max === null) { - if ($min === 0) { - [$name, $value] = ['zero_or_more', '*']; - } elseif ($min === 1) { - [$name, $value] = ['one_or_more', '+']; - } else { - [$name, $value] = ['n_or_more', '{' . $min . ',}']; - } - } elseif ($min === 0 && $max === 1) { - [$name, $value] = ['zero_or_one', '?']; - } elseif ($min === $max) { - [$name, $value] = ['exactly_n', '{' . $min . '}']; - } else { - // also normalizes "{,5}" the walkers could not sscanf - [$name, $value] = ['n_to_m', '{' . $min . ',' . $max . '}']; - } - - return new RegexTreeNode('#quantification', null, array_merge(self::convert($node->node), [ - self::token($name . $suffix, $value), - ])); - } - - private static function convertClass(CharClassNode $node): RegexTreeNode - { - $expression = $node->expression; - $members = $expression instanceof AlternationNode ? $expression->alternatives : [$expression]; - - $children = []; - foreach ($members as $member) { - $children = array_merge($children, self::convertClassMember($member)); - } - - return new RegexTreeNode($node->isNegated ? '#negativeclass' : '#class', null, $children); - } - - /** - * @return list - */ - private static function convertClassMember(NodeInterface $node): array - { - if ($node instanceof RangeNode) { - return [new RegexTreeNode('#range', null, array_merge(self::convert($node->start), self::convert($node->end)))]; - } - - if ($node instanceof PosixClassNode) { - return [self::token('posix_class', '[:' . $node->class . ':]')]; - } - - if ($node instanceof CharTypeNode) { - return [self::token('character_type', '\\' . $node->value)]; - } - - if ($node instanceof UnicodePropNode) { - return [self::token('character_type', self::unicodeProperty($node))]; - } - - if ($node instanceof CharLiteralNode) { - return [self::token('dynamic_character', self::character($node->codePoint))]; - } - - if ($node instanceof ControlCharNode) { - return [self::token('character', self::character($node->codePoint))]; - } - - if ($node instanceof LiteralNode) { - // inside a class every member is a plain literal, "." included - $tokens = []; - foreach (mb_str_split($node->value, encoding: 'UTF-8') as $char) { - $tokens[] = self::token('literal', $char); - } - - return $tokens; - } - - return [self::token('class_member', $node::class)]; - } - - private static function convertConditional(ConditionalNode $node): RegexTreeNode - { - $condition = $node->condition; - - if ($condition instanceof BackrefNode) { - $ref = $condition->ref; - if (ctype_digit($ref)) { - $id = '#absolutecondition'; - } elseif (str_starts_with($ref, '-') || str_starts_with($ref, '+')) { - $id = '#relativecondition'; - } else { - $id = '#namedcondition'; - } - - $children = [$id === '#namedcondition' - ? self::token('capturing_name', $ref) - : self::token('index', $ref)]; - } elseif ($condition instanceof VersionConditionNode) { - $children = [self::token('version_condition', $condition->operator . $condition->version)]; - $id = '#assertioncondition'; - } elseif ($condition instanceof SubroutineNode) { - $children = [self::token('recursion_condition', $condition->reference)]; - $id = '#assertioncondition'; - } else { - $children = array_merge(self::convert($condition)); - $id = '#assertioncondition'; - } - - $body = self::convertConditionBody($node->yes, $node->no); - if ($body !== null) { - $children[] = $body; - } - - return new RegexTreeNode($id, null, $children); - } - - private static function convertConditionBody(NodeInterface $yes, NodeInterface $no): ?RegexTreeNode - { - $yesNode = self::convertBody($yes); - $noNode = self::convertBody($no); - - if ($yesNode === null && $noNode === null) { - return null; - } - - if ($noNode === null) { - return $yesNode; - } - - if ($yesNode === null) { - return $noNode; - } - - return new RegexTreeNode('#alternation', null, [$yesNode, self::token('alternation', '|'), $noNode]); - } - - /** - * @return list - */ - private static function convertDefinition(DefineNode $node): array - { - $children = [self::token('define', 'DEFINE')]; - - $body = self::convertBody($node->content); - if ($body !== null) { - $children[] = $body; - } - - return [new RegexTreeNode('#assertioncondition', null, $children)]; - } - - private static function unicodeProperty(UnicodePropNode $node): string - { - // a caret inside the property already carries the negation - $letter = str_starts_with($node->prop, '^') || str_starts_with($node->prop, '{^') - ? 'p' - : ($node->negatedSyntax ? 'P' : 'p'); - - return '\\' . $letter . $node->prop; - } - - private static function character(int $codePoint): string - { - return mb_chr($codePoint, 'UTF-8') ?? "\x00"; - } - - private static function token(string $name, string $value): RegexTreeNode - { - return new RegexTreeNode('token', ['token' => $name, 'value' => $value, 'namespace' => 'default']); - } - -} diff --git a/src/Type/Regex/RegexGroupParser.php b/src/Type/Regex/RegexGroupParser.php index 64e322b6568..b00df345c61 100644 --- a/src/Type/Regex/RegexGroupParser.php +++ b/src/Type/Regex/RegexGroupParser.php @@ -5,6 +5,33 @@ use Nette\Utils\RegexpException; use Nette\Utils\Strings; use PHPRegex\Parser\Exception\RegexException; +use PHPRegex\Parser\Node\AlternationNode; +use PHPRegex\Parser\Node\AnchorNode; +use PHPRegex\Parser\Node\AssertionNode; +use PHPRegex\Parser\Node\BackrefNode; +use PHPRegex\Parser\Node\CalloutNode; +use PHPRegex\Parser\Node\CharClassNode; +use PHPRegex\Parser\Node\CharLiteralNode; +use PHPRegex\Parser\Node\CharTypeNode; +use PHPRegex\Parser\Node\CommentNode; +use PHPRegex\Parser\Node\ConditionalNode; +use PHPRegex\Parser\Node\ControlCharNode; +use PHPRegex\Parser\Node\DefineNode; +use PHPRegex\Parser\Node\DotNode; +use PHPRegex\Parser\Node\GroupNode; +use PHPRegex\Parser\Node\GroupType; +use PHPRegex\Parser\Node\KeepNode; +use PHPRegex\Parser\Node\LiteralNode; +use PHPRegex\Parser\Node\NodeInterface; +use PHPRegex\Parser\Node\PcreVerbNode; +use PHPRegex\Parser\Node\PosixClassNode; +use PHPRegex\Parser\Node\QuantifierBounds; +use PHPRegex\Parser\Node\QuantifierNode; +use PHPRegex\Parser\Node\RangeNode; +use PHPRegex\Parser\Node\RegexNode; +use PHPRegex\Parser\Node\SequenceNode; +use PHPRegex\Parser\Node\SubroutineNode; +use PHPRegex\Parser\Node\UnicodePropNode; use PHPStan\DependencyInjection\AutowiredService; use PHPStan\Php\PhpVersion; use PHPStan\ShouldNotHappenException; @@ -19,17 +46,11 @@ use PHPStan\Type\TypeCombinator; use PHPStan\Type\UnionType; use function array_key_exists; -use function array_pop; -use function array_values; +use function array_merge; use function count; use function in_array; -use function is_int; -use function rtrim; -use function sscanf; +use function mb_str_split; use function str_contains; -use function str_replace; -use function strlen; -use function substr; use function trim; #[AutowiredService] @@ -44,13 +65,13 @@ final class RegexGroupParser // to avoid combinatorial explosion from nested optional/bounded quantifications private const LITERALS_LIMIT = 100; - /** @var array */ + /** @var array */ private static array $parsedAst = []; public function __construct( private PhpVersion $phpVersion, private RegexExpressionHelper $regexExpressionHelper, - private PHPRegexTreeNodeBuilder $treeNodeBuilder, + private PHPRegexReader $reader, ) { } @@ -83,12 +104,9 @@ public function parseGroups(string $regex): ?RegexAstWalkResult } } - // freespacing comments are read by the mode-aware parser itself, - // class- and \Q-aware, so nothing is stripped up front - $rawRegex = $this->regexExpressionHelper->removeDelimitersAndModifiers($regex); try { - $ast = self::$parsedAst[$cacheKey] = $this->treeNodeBuilder->parse( + $ast = self::$parsedAst[$cacheKey] = $this->reader->read( $rawRegex, $modifiers, self::delimiterOf($regex), @@ -98,16 +116,13 @@ public function parseGroups(string $regex): ?RegexAstWalkResult } } - $this->updateAlternationAstRemoveVerticalBarsAndAddEmptyToken($ast); - $this->updateCapturingAstAddEmptyToken($ast); - $captureOnlyNamed = false; if ($this->phpVersion->supportsPregCaptureOnlyNamedGroups()) { $captureOnlyNamed = str_contains($modifiers, 'n'); } $astWalkResult = $this->walkRegexAst( - $ast, + $ast->pattern, null, 0, false, @@ -119,13 +134,13 @@ public function parseGroups(string $regex): ?RegexAstWalkResult ); $subjectAsGroupResult = $this->walkGroupAst( - $ast, + $ast->pattern, false, $modifiers, RegexGroupWalkResult::createEmpty(), ); - if (!$subjectAsGroupResult->mightContainEmptyStringLiteral() && !$this->containsEscapeK($ast)) { + if (!$subjectAsGroupResult->mightContainEmptyStringLiteral() && !$this->containsEscapeK($ast->pattern)) { // we could handle numeric-string, in case we know the regex is delimited by ^ and $ if ($subjectAsGroupResult->isNonFalsy()->yes()) { $astWalkResult = $astWalkResult->withSubjectBaseType( @@ -141,11 +156,6 @@ public function parseGroups(string $regex): ?RegexAstWalkResult return $astWalkResult; } - private function createEmptyTokenTreeNode(): RegexTreeNode - { - return new RegexTreeNode('token', ['token' => 'literal', 'value' => '', 'namespace' => 'default']); - } - private static function delimiterOf(string $regex): string { $trimmed = trim($regex); @@ -153,53 +163,13 @@ private static function delimiterOf(string $regex): string return $trimmed === '' ? '/' : $trimmed[0]; } - private function updateAlternationAstRemoveVerticalBarsAndAddEmptyToken(RegexTreeNode $ast): void - { - $children = $ast->getChildren(); - - foreach ($children as $i => $child) { - $this->updateAlternationAstRemoveVerticalBarsAndAddEmptyToken($child); - - if ($ast->getId() !== '#alternation' || $child->getValueToken() !== 'alternation') { - continue; - } - - unset($children[$i]); - - if ($i !== 0 - && isset($children[$i + 1]) - && $children[$i + 1]->getValueToken() !== 'alternation') { - continue; - } - - $children[$i] = $this->createEmptyTokenTreeNode(); - } - - $ast->setChildren(array_values($children)); - } - - private function updateCapturingAstAddEmptyToken(RegexTreeNode $ast): void - { - foreach ($ast->getChildren() as $child) { - $this->updateCapturingAstAddEmptyToken($child); - } - - if ($ast->getId() !== '#capturing' || $ast->getChildren() !== []) { - return; - } - - $emptyAlternationAst = new RegexTreeNode('#alternation'); - $emptyAlternationAst->setChildren([$this->createEmptyTokenTreeNode()]); - $ast->setChildren([$emptyAlternationAst]); - } - - private function containsEscapeK(RegexTreeNode $ast): bool + private function containsEscapeK(NodeInterface $node): bool { - if ($ast->getId() === 'token' && $ast->getValueToken() === 'match_point_reset') { + if ($node instanceof KeepNode) { return true; } - foreach ($ast->getChildren() as $child) { + foreach ($node->getChildren() as $child) { if ($this->containsEscapeK($child)) { return true; } @@ -209,7 +179,7 @@ private function containsEscapeK(RegexTreeNode $ast): bool } private function walkRegexAst( - RegexTreeNode $ast, + NodeInterface $ast, ?RegexAlternation $alternation, int $combinationIndex, bool $inOptionalQuantification, @@ -221,77 +191,81 @@ private function walkRegexAst( ): RegexAstWalkResult { $group = null; - if ($ast->getId() === '#capturing') { - $astWalkResult = $astWalkResult->nextCaptureGroupId(); - - $group = new RegexCapturingGroup( - $astWalkResult->getCaptureGroupId(), - null, - $alternation, - $inOptionalQuantification, - $parentGroup, - $this->createGroupType( - $ast, - $this->allowConstantTypes($patternModifiers, $repeatedMoreThanOnce, $parentGroup), - $patternModifiers, - ), - ); - $parentGroup = $group; - } elseif ($ast->getId() === '#namedcapturing') { - $astWalkResult = $astWalkResult->nextCaptureGroupId(); - - $name = $ast->getChild(0)->getValueValue(); - $group = new RegexCapturingGroup( - $astWalkResult->getCaptureGroupId(), - $name, - $alternation, - $inOptionalQuantification, - $parentGroup, - $this->createGroupType( - $ast, - $this->allowConstantTypes($patternModifiers, $repeatedMoreThanOnce, $parentGroup), - $patternModifiers, - ), - ); - $parentGroup = $group; - } elseif ($ast->getId() === '#noncapturing') { - $group = new RegexNonCapturingGroup( - $alternation, - $inOptionalQuantification, - $parentGroup, - false, - ); - $parentGroup = $group; - } elseif ($ast->getId() === '#noncapturingreset') { - $group = new RegexNonCapturingGroup( - $alternation, - $inOptionalQuantification, - $parentGroup, - true, - ); - $parentGroup = $group; + if ($ast instanceof GroupNode) { + if ($ast->type === GroupType::Capturing()) { + $astWalkResult = $astWalkResult->nextCaptureGroupId(); + + $group = new RegexCapturingGroup( + $astWalkResult->getCaptureGroupId(), + null, + $alternation, + $inOptionalQuantification, + $parentGroup, + $this->createGroupType( + $ast, + $this->allowConstantTypes($patternModifiers, $repeatedMoreThanOnce, $parentGroup), + $patternModifiers, + ), + ); + $parentGroup = $group; + } elseif ($ast->type === GroupType::Named()) { + $astWalkResult = $astWalkResult->nextCaptureGroupId(); + + $group = new RegexCapturingGroup( + $astWalkResult->getCaptureGroupId(), + $ast->name, + $alternation, + $inOptionalQuantification, + $parentGroup, + $this->createGroupType( + $ast, + $this->allowConstantTypes($patternModifiers, $repeatedMoreThanOnce, $parentGroup), + $patternModifiers, + ), + ); + $parentGroup = $group; + } elseif ($ast->type === GroupType::BranchReset()) { + $group = new RegexNonCapturingGroup( + $alternation, + $inOptionalQuantification, + $parentGroup, + true, + ); + $parentGroup = $group; + } elseif ($ast->type !== GroupType::ScanSubstring()) { + $group = new RegexNonCapturingGroup( + $alternation, + $inOptionalQuantification, + $parentGroup, + false, + ); + $parentGroup = $group; + } } $inOptionalQuantification = false; - if ($ast->getId() === '#quantification') { - [$min, $max] = $this->getQuantificationRange($ast); + if ($ast instanceof QuantifierNode) { + $bounds = QuantifierBounds::parse($ast->quantifier); + if ($bounds === null) { + throw new ShouldNotHappenException(); + } - if ($min === 0) { + if ($bounds->min === 0) { $inOptionalQuantification = true; } - if ($max === null || $max > 1) { + if ($bounds->max === null || $bounds->max > 1) { $repeatedMoreThanOnce = true; } } - if ($ast->getId() === '#alternation') { + if ($ast instanceof AlternationNode) { $astWalkResult = $astWalkResult->nextAlternationId(); - $alternation = new RegexAlternation($astWalkResult->getAlternationId(), count($ast->getChildren())); + $alternation = new RegexAlternation($astWalkResult->getAlternationId(), count($ast->alternatives)); } - if ($ast->getId() === '#mark') { - return $astWalkResult->markVerb($ast->getChild(0)->getValueValue()); + if ($ast instanceof PcreVerbNode && str_starts_with($ast->verb, 'MARK:')) { + return $astWalkResult->markVerb(substr($ast->verb, 5)); } if ( @@ -305,9 +279,11 @@ private function walkRegexAst( } } - foreach ($ast->getChildren() as $child) { + // a conditional holds its yes and no branches as the engine reads + // them: the groups they carry sit in one alternation + if ($ast instanceof ConditionalNode || $ast instanceof DefineNode) { $astWalkResult = $this->walkRegexAst( - $child, + $ast instanceof ConditionalNode ? $ast->condition : $ast->content, $alternation, $combinationIndex, $inOptionalQuantification, @@ -317,12 +293,46 @@ private function walkRegexAst( $patternModifiers, $astWalkResult, ); + } - if ($ast->getId() !== '#alternation') { - continue; - } + $branches = $ast instanceof AlternationNode + ? $ast->alternatives + : $ast->getChildren(); + + foreach ($branches as $i => $branch) { + $astWalkResult = $this->walkRegexAst( + $branch, + $alternation, + $ast instanceof AlternationNode ? $i : $combinationIndex, + $inOptionalQuantification, + $parentGroup, + $captureOnlyNamed, + $repeatedMoreThanOnce, + $patternModifiers, + $astWalkResult, + ); + } - $combinationIndex++; + if ($ast instanceof ConditionalNode || $ast instanceof DefineNode) { + $yesNo = $ast instanceof ConditionalNode + ? [$ast->yes, $ast->no] + : [$ast->content]; + $astWalkResult = $astWalkResult->nextAlternationId(); + $conditionAlternation = new RegexAlternation($astWalkResult->getAlternationId(), count($yesNo)); + + foreach ($yesNo as $i => $branch) { + $astWalkResult = $this->walkRegexAst( + $branch, + $conditionAlternation, + $i, + $inOptionalQuantification, + $parentGroup, + $captureOnlyNamed, + $repeatedMoreThanOnce, + $patternModifiers, + $astWalkResult, + ); + } } return $astWalkResult; @@ -351,69 +361,25 @@ private function allowConstantTypes( return true; } - /** @return array{?int, ?int} */ - private function getQuantificationRange(RegexTreeNode $node): array - { - if ($node->getId() !== '#quantification') { - throw new ShouldNotHappenException(); - } - - $min = null; - $max = null; - - $lastChild = $node->getChild($node->getChildrenNumber() - 1); - $value = $lastChild->getValue(); - - // normalize away possessive and lazy quantifier-modifiers - $token = str_replace(['_possessive', '_lazy'], '', $value['token']); - $value = rtrim($value['value'], '+?'); - - if ($token === 'n_to_m') { - if (sscanf($value, '{%d,%d}', $n, $m) !== 2 || !is_int($n) || !is_int($m)) { - throw new ShouldNotHappenException(); - } - - $min = $n; - $max = $m; - } elseif ($token === 'n_or_more') { - if (sscanf($value, '{%d,}', $n) !== 1 || !is_int($n)) { - throw new ShouldNotHappenException(); - } - - $min = $n; - } elseif ($token === 'exactly_n') { - if (sscanf($value, '{%d}', $n) !== 1 || !is_int($n)) { - throw new ShouldNotHappenException(); - } - - $min = $n; - $max = $n; - } elseif ($token === 'zero_or_one') { - $min = 0; - $max = 1; - } elseif ($token === 'zero_or_more') { - $min = 0; - } elseif ($token === 'one_or_more') { - $min = 1; - } - - return [$min, $max]; - } - - private function createGroupType(RegexTreeNode $group, bool $maybeConstant, string $patternModifiers): Type + private function createGroupType(GroupNode $group, bool $maybeConstant, string $patternModifiers): Type { $rootAlternation = $this->getRootAlternation($group); if ($rootAlternation !== null) { $types = []; - foreach ($rootAlternation->getChildren() as $alternative) { - $types[] = $this->createGroupType($alternative, $maybeConstant, $patternModifiers); + foreach ($rootAlternation->alternatives as $alternative) { + $types[] = $this->createGroupTypeFromNode($alternative, $maybeConstant, $patternModifiers); } return TypeCombinator::union(...$types); } + return $this->createGroupTypeFromNode($group->child, $maybeConstant, $patternModifiers); + } + + private function createGroupTypeFromNode(NodeInterface $node, bool $maybeConstant, string $patternModifiers): Type + { $walkResult = $this->walkGroupAst( - $group, + $node, false, $patternModifiers, RegexGroupWalkResult::createEmpty(), @@ -451,30 +417,17 @@ private function createGroupType(RegexTreeNode $group, bool $maybeConstant, stri return new StringType(); } - private function getRootAlternation(RegexTreeNode $group): ?RegexTreeNode + private function getRootAlternation(GroupNode $group): ?AlternationNode { - if ( - $group->getId() === '#capturing' - && count($group->getChildren()) === 1 - && $group->getChild(0)->getId() === '#alternation' - ) { - return $group->getChild(0); - } - - // 1st token within a named capturing group is a token holding the group-name - if ( - $group->getId() === '#namedcapturing' - && count($group->getChildren()) === 2 - && $group->getChild(1)->getId() === '#alternation' - ) { - return $group->getChild(1); + if ($group->child instanceof AlternationNode) { + return $group->child; } return null; } private function walkGroupAst( - RegexTreeNode $ast, + NodeInterface $ast, bool $inClass, string $patternModifiers, RegexGroupWalkResult $walkResult, @@ -484,7 +437,7 @@ private function walkGroupAst( $quantifiedLiterals = null; if ( - $ast->getId() === '#concatenation' + $ast instanceof SequenceNode && count($children) > 0 && !$walkResult->isInOptionalQuantification() ) { @@ -514,18 +467,19 @@ private function walkGroupAst( $walkResult = $walkResult->nonFalsy(TrinaryLogic::createYes()); } } - } elseif ($ast->getId() === '#quantification') { - [$min, $max] = $this->getQuantificationRange($ast); + } elseif ($ast instanceof QuantifierNode) { + $bounds = QuantifierBounds::parse($ast->quantifier); + [$min, $max] = $bounds === null ? [null, null] : [$bounds->min, $bounds->max]; if ($min === 0) { $walkResult = $walkResult->inOptionalQuantification(true); } if (!$walkResult->isInOptionalQuantification()) { - if ($min >= 1) { + if ($min !== null && $min >= 1) { $walkResult = $walkResult->nonEmpty(TrinaryLogic::createYes()); } - if ($min >= 2) { + if ($min !== null && $min >= 2) { $walkResult = $walkResult->nonFalsy(TrinaryLogic::createYes()); } } @@ -535,20 +489,23 @@ private function walkGroupAst( $quantifiedLiterals = $this->getQuantifiedLiterals($ast, $min, $max, $inClass, $patternModifiers, $walkResult); $walkResult = $walkResult->onlyLiterals(null); - } elseif ($ast->getId() === '#class' && $walkResult->getOnlyLiterals() !== null) { + } elseif ($ast instanceof CharClassNode && !$ast->isNegated && $walkResult->getOnlyLiterals() !== null) { $inClass = true; $newLiterals = []; - foreach ($children as $child) { + $members = $ast->expression instanceof AlternationNode + ? $ast->expression->alternatives + : [$ast->expression]; + foreach ($this->classMembers($members) as $member) { $oldLiterals = $walkResult->getOnlyLiterals(); - $this->getLiteralValue($child, $oldLiterals, true, $patternModifiers, true); + $this->getLiteralValue($member, $oldLiterals, true, $patternModifiers, true); foreach ($oldLiterals ?? [] as $oldLiteral) { $newLiterals[] = $oldLiteral; } } $walkResult = $walkResult->onlyLiterals($newLiterals); - } elseif ($ast->getId() === 'token') { + } elseif ($ast instanceof LiteralNode) { $onlyLiterals = $walkResult->getOnlyLiterals(); $literalValue = $this->getLiteralValue($ast, $onlyLiterals, !$inClass, $patternModifiers, false); $walkResult = $walkResult->onlyLiterals($onlyLiterals); @@ -564,12 +521,34 @@ private function walkGroupAst( $walkResult = $walkResult->nonEmpty(TrinaryLogic::createYes()); } } - } elseif (!in_array($ast->getId(), ['#capturing', '#namedcapturing', '#alternation'], true)) { + } elseif ( + !($ast instanceof GroupNode + && ($ast->type === GroupType::Capturing() || $ast->type === GroupType::Named())) + && !$ast instanceof AlternationNode + && !$ast instanceof LiteralNode + ) { + // every other leaf and group the walk does not fold into constants + // drops them: an escape, a wildcard, a backreference, a verb — + // even a \d that reads as '0' — just as every token outside the + // literal family did + $literalValue = $this->getLiteralValue($ast, $onlyLiterals, !$inClass, $patternModifiers, false); $walkResult = $walkResult->onlyLiterals(null); + + if ($literalValue !== null) { + if (Strings::match($literalValue, '/^\d+$/') === null) { + $walkResult = $walkResult->numeric(TrinaryLogic::createNo()); + } elseif ($walkResult->isNumeric()->maybe()) { + $walkResult = $walkResult->numeric(TrinaryLogic::createYes()); + } + + if (!$walkResult->isInOptionalQuantification() && $literalValue !== '') { + $walkResult = $walkResult->nonEmpty(TrinaryLogic::createYes()); + } + } } - if ($ast->getId() === '#alternation') { - if (count($children) === 0) { + if ($ast instanceof AlternationNode) { + if (count($ast->alternatives) === 0) { return $walkResult; } @@ -580,9 +559,9 @@ private function walkGroupAst( $nonEmpty = TrinaryLogic::createYes(); $nonFalsy = TrinaryLogic::createYes(); $numeric = TrinaryLogic::createYes(); - foreach ($children as $child) { + foreach ($ast->alternatives as $alternative) { $childResult = $this->walkGroupAst( - $child, + $alternative, $inClass, $patternModifiers, $walkResult->onlyLiterals([]) @@ -626,11 +605,20 @@ private function walkGroupAst( // [^0-9] should not parse as numeric-string, and [^list-everything-but-numbers] is technically // doable but really silly compared to just \d so we can safely assume the string is not numeric // for negative classes - if ($ast->getId() === '#negativeclass') { + if ($ast instanceof CharClassNode && $ast->isNegated) { $walkResult = $walkResult->numeric(TrinaryLogic::createNo()); } - foreach ($children as $child) { + // the members of a class the combination above already folded walk one + // by one below, as they did before, without the wrapper that groups + // them in the tree + $descend = $ast instanceof CharClassNode + ? $this->classMembers($ast->expression instanceof AlternationNode + ? $ast->expression->alternatives + : [$ast->expression]) + : $children; + + foreach ($descend as $child) { $walkResult = $this->walkGroupAst( $child, $inClass, @@ -639,8 +627,8 @@ private function walkGroupAst( ); } - if ($ast->getId() === '#quantification') { - // the bottom walk above nulls the literals via the quantifier token, + if ($ast instanceof QuantifierNode) { + // the bottom walk above nulls the literals via the quantifier node, // so restore the literals enumerated up-front for bounded quantifications $walkResult = $walkResult->onlyLiterals($quantifiedLiterals); } @@ -655,22 +643,17 @@ private function walkGroupAst( * * @return array|null */ - private function getQuantifiedLiterals(RegexTreeNode $ast, ?int $min, ?int $max, bool $inClass, string $patternModifiers, RegexGroupWalkResult $walkResult): ?array + private function getQuantifiedLiterals(QuantifierNode $ast, ?int $min, ?int $max, bool $inClass, string $patternModifiers, RegexGroupWalkResult $walkResult): ?array { $prefixLiterals = $walkResult->getOnlyLiterals(); if ($prefixLiterals === null || $min === null || $max === null) { return null; } - // walk the quantified atom standalone (everything but the trailing quantifier token); - // the atom itself is not optional, so reset the flag to let concatenations accumulate literals - $atomChildren = $ast->getChildren(); - array_pop($atomChildren); - + // walk the quantified atom standalone; the atom itself is not optional, + // so reset the flag to let concatenations accumulate literals $atomResult = $walkResult->onlyLiterals([])->inOptionalQuantification(false); - foreach ($atomChildren as $atomChild) { - $atomResult = $this->walkGroupAst($atomChild, $inClass, $patternModifiers, $atomResult); - } + $atomResult = $this->walkGroupAst($ast->node, $inClass, $patternModifiers, $atomResult); $atomLiterals = $atomResult->getOnlyLiterals(); if ($atomLiterals === null) { @@ -741,12 +724,13 @@ private function repeatLiterals(array $literals, int $min, int $max): ?array return array_values($collected); } - private function isMaybeEmptyNode(RegexTreeNode $node, string $patternModifiers, bool &$isNonFalsy): bool + private function isMaybeEmptyNode(NodeInterface $node, string $patternModifiers, bool &$isNonFalsy): bool { - if ($node->getId() === '#quantification') { - [$min] = $this->getQuantificationRange($node); + if ($node instanceof QuantifierNode) { + $bounds = QuantifierBounds::parse($node->quantifier); + $min = $bounds === null ? null : $bounds->min; - if ($min > 0) { + if ($min !== null && $min > 0) { return false; } @@ -765,12 +749,12 @@ private function isMaybeEmptyNode(RegexTreeNode $node, string $patternModifiers, // an alternation is maybe-empty if any of its branches is maybe-empty, // and non-falsy only if every branch is non-falsy - if ($node->getId() === '#alternation') { + if ($node instanceof AlternationNode) { $maybeEmpty = false; $allNonFalsy = true; - foreach ($node->getChildren() as $child) { + foreach ($node->alternatives as $alternative) { $childNonFalsy = false; - if ($this->isMaybeEmptyNode($child, $patternModifiers, $childNonFalsy)) { + if ($this->isMaybeEmptyNode($alternative, $patternModifiers, $childNonFalsy)) { $maybeEmpty = true; } $allNonFalsy = $allNonFalsy && $childNonFalsy; @@ -793,99 +777,119 @@ private function isMaybeEmptyNode(RegexTreeNode $node, string $patternModifiers, } /** - * @param array|null $onlyLiterals + * The members a character class enumerates, one array entry per + * character, as the engine reads them. A range keeps its own node: it + * enumerates no constant of its own and drops the constants around it, + * as it always did. + * + * @param list $members + * @return list */ - private function getLiteralValue(RegexTreeNode $node, ?array &$onlyLiterals, bool $appendLiterals, string $patternModifiers, bool $inCharacterClass): ?string + private function classMembers(array $members): array { - if ($node->getId() !== 'token') { - return null; + $result = []; + foreach ($members as $member) { + if ($member instanceof LiteralNode) { + foreach (mb_str_split($member->value, encoding: 'UTF-8') as $char) { + $result[] = new LiteralNode($char, $member->getStartPosition(), $member->getEndPosition()); + } + continue; + } + + $result[] = $member; } - // token is the token name from grammar without the namespace so literal and class:literal are both called literal here - $token = $node->getValueToken(); - $value = $node->getValueValue(); + return $result; + } - if ( - in_array($token, [ - 'literal', - // literal "-" in front/back of a character class like '[-a-z]' or '[abc-]', not forming a range - 'range', - // literal "[" or "]" inside character classes '[[]' or '[]]' - 'class_', '_class', - ], true) - ) { - if (str_contains($patternModifiers, 'x') && trim($value) === '') { + /** + * @param array|null $onlyLiterals + */ + private function getLiteralValue(NodeInterface $node, ?array &$onlyLiterals, bool $appendLiterals, string $patternModifiers, bool $inCharacterClass): ?string + { + if ($node instanceof LiteralNode) { + if (str_contains($patternModifiers, 'x') && trim($node->value) === '') { return null; } - $isEscaped = false; - if (strlen($value) > 1 && $value[0] === '\\') { - $value = substr($value, 1) ?: ''; - $isEscaped = true; - } + // one literal per character, as the walk counts tokens, the whole + // run back as the value; an empty run reads as the empty string, + // as the empty branch of an alternation always did; a literal + // "." outside a class is a constant, the wildcard is its own node + $chars = $node->value === '' ? [''] : mb_str_split($node->value, encoding: 'UTF-8'); - if ( - $appendLiterals - && $onlyLiterals !== null - ) { - if ( - in_array($value, ['.'], true) - && !($isEscaped || $inCharacterClass) - ) { - $onlyLiterals = null; - } else { + if ($appendLiterals && $onlyLiterals !== null) { + foreach ($chars as $char) { if ($onlyLiterals === []) { - $onlyLiterals = [$value]; + $onlyLiterals = [$char]; } else { foreach ($onlyLiterals as &$literal) { - $literal .= $value; + $literal .= $char; } } } } - return $value; + return $node->value; } - if (!in_array($token, ['capturing_name'], true)) { + if ($node instanceof DotNode) { + // the wildcard matches anything: no constant literals survive it $onlyLiterals = null; + + return '.'; } - // character escape sequences, just return a fixed string - if (in_array($token, ['character', 'dynamic_character', 'character_type'], true)) { - if ($token === 'character_type' && $value === '\d') { + if ($node instanceof CharTypeNode) { + if (!$inCharacterClass && $node->value === 'd') { return '0'; } - return $value; + return '\\' . $node->value; } - // [:digit:] and the like, more support coming later - if ($token === 'posix_class') { - if ($value === '[:digit:]') { + if ($node instanceof UnicodePropNode) { + return '\\' . ($node->negatedSyntax ? 'P' : 'p') . $node->prop; + } + + if ($node instanceof CharLiteralNode) { + return mb_chr($node->codePoint, 'UTF-8') ?? ''; + } + + if ($node instanceof ControlCharNode) { + return mb_chr($node->codePoint, 'UTF-8') ?? ''; + } + + // [:digit:] and the like + if ($node instanceof PosixClassNode) { + if ($node->class === 'digit') { return '0'; } - if (in_array($value, ['[:alpha:]', '[:alnum:]', '[:upper:]', '[:lower:]', '[:word:]', '[:ascii:]', '[:print:]', '[:xdigit:]', '[:graph:]'], true)) { + if (in_array($node->class, ['alpha', 'alnum', 'upper', 'lower', 'word', 'ascii', 'print', 'xdigit', 'graph'], true)) { return 'a'; } - if ($value === '[:blank:]') { + if ($node->class === 'blank') { return " \t"; } - if ($value === '[:cntrl:]') { + if ($node->class === 'cntrl') { return "\x00\x1F"; } - if ($value === '[:space:]') { + if ($node->class === 'space') { return " \t\r\n\v\f"; } - if ($value === '[:punct:]') { + if ($node->class === 'punct') { return '!"#$%&\'()*+,\-./:;<=>?@[\]^_`{|}~'; } } - if (in_array($token, ['anchor', 'match_point_reset'], true)) { + if ($node instanceof AnchorNode || $node instanceof AssertionNode || $node instanceof KeepNode) { return ''; } + // backreferences, subroutine calls, verbs, callouts: their content is + // not a constant, and no constant literals survive them + $onlyLiterals = null; + return null; } diff --git a/src/Type/Regex/RegexTreeNode.php b/src/Type/Regex/RegexTreeNode.php deleted file mode 100644 index 31568f337d0..00000000000 --- a/src/Type/Regex/RegexTreeNode.php +++ /dev/null @@ -1,78 +0,0 @@ - $children - */ - public function __construct( - private readonly string $id, - private readonly ?array $value = null, - private array $children = [], - ) { - } - - public function getId(): string - { - return $this->id; - } - - /** - * @return array{token?: string, value?: string, namespace?: string}|null - */ - public function getValue(): ?array - { - return $this->value; - } - - public function getValueToken(): ?string - { - return $this->value['token'] ?? null; - } - - public function getValueValue(): ?string - { - return $this->value['value'] ?? null; - } - - /** - * @return list - */ - public function getChildren(): array - { - return $this->children; - } - - /** - * @param list $children - */ - public function setChildren(array $children): void - { - $this->children = $children; - } - - public function getChild(int $i): ?self - { - return $this->children[$i] ?? null; - } - - public function getChildrenNumber(): int - { - return count($this->children); - } - -} diff --git a/tests/PHPStan/Command/IgnoredRegexValidatorTest.php b/tests/PHPStan/Command/IgnoredRegexValidatorTest.php index e31c6147d3e..99bfdd5f970 100644 --- a/tests/PHPStan/Command/IgnoredRegexValidatorTest.php +++ b/tests/PHPStan/Command/IgnoredRegexValidatorTest.php @@ -6,7 +6,7 @@ use PHPStan\Php\PhpVersion; use PHPStan\PhpDoc\TypeStringResolver; use PHPStan\Testing\PHPStanTestCase; -use PHPStan\Type\Regex\PHPRegexTreeNodeBuilder; +use PHPStan\Type\Regex\PHPRegexReader; use PHPUnit\Framework\Attributes\DataProvider; class IgnoredRegexValidatorTest extends PHPStanTestCase @@ -166,7 +166,7 @@ public function testValidate( { $validator = new IgnoredRegexValidator( self::getContainer()->getByType(TypeStringResolver::class), - new PHPRegexTreeNodeBuilder(new PhpVersion(PHP_VERSION_ID), PcreVersion::fromRuntime()), + new PHPRegexReader(new PhpVersion(PHP_VERSION_ID), PcreVersion::fromRuntime()), ); $result = $validator->validate($regex); diff --git a/tests/PHPStan/Type/Regex/RegexShapeReadingTest.php b/tests/PHPStan/Type/Regex/RegexShapeReadingTest.php index c59686f44a2..1eca4c7c44b 100644 --- a/tests/PHPStan/Type/Regex/RegexShapeReadingTest.php +++ b/tests/PHPStan/Type/Regex/RegexShapeReadingTest.php @@ -37,7 +37,7 @@ public function testMultiByteLiteralsSplitAsUtf8WhateverTheDefaultEncoding(): vo $parser = new RegexGroupParser( new PhpVersion(PHP_VERSION_ID), $helper, - new PHPRegexTreeNodeBuilder(new PhpVersion(PHP_VERSION_ID), PcreVersion::fromRuntime()), + new PHPRegexReader(new PhpVersion(PHP_VERSION_ID), PcreVersion::fromRuntime()), ); $this->resetParsedAstCache(); @@ -56,7 +56,7 @@ private function createParser(): RegexGroupParser return new RegexGroupParser( new PhpVersion(PHP_VERSION_ID), $helper, - new PHPRegexTreeNodeBuilder(new PhpVersion(PHP_VERSION_ID), PcreVersion::fromRuntime()), + new PHPRegexReader(new PhpVersion(PHP_VERSION_ID), PcreVersion::fromRuntime()), ); } @@ -265,24 +265,29 @@ public function testNoAutoCaptureOptionDropsTheGroups(): void self::assertSame([], $result->getCapturingGroups(), 'the walk must not report a capturing group'); } - public function testTreeNodeCarriesTheWalkersSurface(): void + public function testTheReaderHandsBackTheParserOwnTree(): void { - $leaf = new RegexTreeNode('token', ['token' => 'literal', 'value' => 'a', 'namespace' => 'default']); - $root = new RegexTreeNode('#concatenation', null, [$leaf]); - - self::assertSame('#concatenation', $root->getId()); - self::assertNull($root->getValueToken()); - self::assertSame('literal', $leaf->getValueToken()); - self::assertSame('a', $leaf->getValueValue()); - self::assertSame($leaf, $root->getChild(0)); - self::assertNull($root->getChild(1)); - self::assertSame(1, $root->getChildrenNumber()); - self::assertCount(1, $root->getChildren()); - - $replaced = new RegexTreeNode('token', ['token' => 'literal', 'value' => '']); - $root->setChildren([$leaf, $replaced]); - self::assertSame(2, $root->getChildrenNumber()); - self::assertSame('default', $root->getChildren()[0]->getValue()['namespace'] ?? null); + // no translation layer stands between the walkers and the parser's + // AST: what read() returns is what the parser built, walked as is + $reader = new PHPRegexReader(new PhpVersion(80400), PcreVersion::fromString('10.44')); + + $tree = $reader->read('(a)', '', '/'); + self::assertInstanceOf(\PHPRegex\Parser\Node\RegexNode::class, $tree); + $body = $tree->pattern; + self::assertInstanceOf(\PHPRegex\Parser\Node\GroupNode::class, $body); + self::assertSame(\PHPRegex\Parser\Node\GroupType::Capturing(), $body->type); + self::assertInstanceOf(\PHPRegex\Parser\Node\LiteralNode::class, $body->child); + self::assertSame('a', $body->child->value); + + $tree = $reader->read('a|b', '', '/'); + self::assertInstanceOf(\PHPRegex\Parser\Node\AlternationNode::class, $tree->pattern); + self::assertCount(2, $tree->pattern->alternatives); + + $tree = $reader->read('a{2,4}', '', '/'); + self::assertInstanceOf(\PHPRegex\Parser\Node\QuantifierNode::class, $tree->pattern); + $bounds = \PHPRegex\Parser\Node\QuantifierBounds::parse($tree->pattern->quantifier); + self::assertSame(2, $bounds->min); + self::assertSame(4, $bounds->max); } public function testDuplicateGroupNamesKeepTheirShapesInlineAndStayRefusedTrailing(): void @@ -316,8 +321,8 @@ public function testTheAnalysedPhpVersionShapesTheEscape(): void $helper = (new ReflectionClass(RegexExpressionHelper::class))->newInstanceWithoutConstructor(); // the engines each judged PHP bundles, pinned: the test says the same // thing on every machine, whatever engine the runner links - $php74 = new RegexGroupParser(new PhpVersion(70400), $helper, new PHPRegexTreeNodeBuilder(new PhpVersion(70400), PcreVersion::fromString('10.33'))); - $php84 = new RegexGroupParser(new PhpVersion(80400), $helper, new PHPRegexTreeNodeBuilder(new PhpVersion(80400), PcreVersion::fromString('10.44'))); + $php74 = new RegexGroupParser(new PhpVersion(70400), $helper, new PHPRegexReader(new PhpVersion(70400), PcreVersion::fromString('10.33'))); + $php84 = new RegexGroupParser(new PhpVersion(80400), $helper, new PHPRegexReader(new PhpVersion(80400), PcreVersion::fromString('10.44'))); $this->resetParsedAstCache(); $result74 = $php74->parseGroups('/(a\x{ 41 })/'); @@ -335,23 +340,35 @@ public function testTheAnalysedPhpVersionShapesTheEscape(): void self::assertNotInstanceOf(ConstantStringType::class, $group84, '8.x reads the padded escape'); // the reader is a service carrying its own target: no statics, no - // environment, one instance per judged PHP - $reader74 = new PHPRegexTreeNodeBuilder(new PhpVersion(70400), PcreVersion::fromString('10.33')); - $reader84 = new PHPRegexTreeNodeBuilder(new PhpVersion(80400), PcreVersion::fromString('10.44')); + // environment, one instance per judged PHP, and it hands back the + // parser's own tree, not a translation of it + $reader74 = new PHPRegexReader(new PhpVersion(70400), PcreVersion::fromString('10.33')); + $reader84 = new PHPRegexReader(new PhpVersion(80400), PcreVersion::fromString('10.44')); - $tree74 = $reader74->parse('a\x{ 41 }', '', '/'); - $tree84 = $reader84->parse('a\x{ 41 }', '', '/'); + $tree74 = $reader74->read('a\x{ 41 }', '', '/')->pattern; + $tree84 = $reader84->read('a\x{ 41 }', '', '/')->pattern; - $fingerprint = static function (RegexTreeNode $n) use (&$fingerprint): string { + $fingerprint = static function (\PHPRegex\Parser\Node\NodeInterface $n) use (&$fingerprint): string { $children = ''; foreach ($n->getChildren() as $child) { $children .= $fingerprint($child); } - return sprintf('(%s|%s|%s)', $n->getId(), (string) $n->getValueToken(), (string) $n->getValueValue()) . $children; + $name = (new ReflectionClass($n))->getShortName(); + $detail = ''; + foreach (['value', 'quantifier', 'type', 'isNegated'] as $prop) { + $propValue = $n->$prop ?? null; + if ($propValue !== null && (is_string($propValue) || is_int($propValue) || is_bool($propValue))) { + $detail .= '|' . $propValue; + } elseif ($propValue !== null) { + $detail .= '|<' . (new ReflectionClass($propValue))->getShortName() . '>'; + } + } + + return sprintf('(%s%s)', $name, $detail) . $children; }; - self::assertSame($fingerprint($tree74), $fingerprint($reader74->parse('a\x{ 41 }', '', '/')), 'the same target reads the same tree twice'); + self::assertSame($fingerprint($tree74), $fingerprint($reader74->read('a\x{ 41 }', '', '/')->pattern), 'the same target reads the same tree twice'); self::assertNotSame($fingerprint($tree74), $fingerprint($tree84), 'the two targets build two trees'); } @@ -416,14 +433,14 @@ public function testAPinAboveTheBundledEngineIsIgnored(): void // the analysed PHP bundles its own engine; config or composer.json // can only narrow below it, never raise the judged engine above // what that PHP can run - $targetOf = static function (PHPRegexTreeNodeBuilder $builder): string { - $builder->parse('a', '', '/'); + $targetOf = static function (PHPRegexReader $reader): string { + $reader->read('a', '', '/'); - $property = new ReflectionProperty(PHPRegexTreeNodeBuilder::class, 'parser'); + $property = new ReflectionProperty(PHPRegexReader::class, 'parser'); $property->setAccessible(true); /** @var \PHPRegex\Parser\RegexParser $parser */ - $parser = $property->getValue($builder); + $parser = $property->getValue($reader); return $parser->target()->pcreVersion; }; @@ -431,9 +448,9 @@ public function testAPinAboveTheBundledEngineIsIgnored(): void // pinned, not read from the runner's engine: the test says the same // thing on every machine $runtimeEngine = PcreVersion::fromString('10.49'); - self::assertSame('10.44', $targetOf(new PHPRegexTreeNodeBuilder(new PhpVersion(80400), $runtimeEngine)), 'a pin above the bundled engine is ignored'); - self::assertSame('10.42', $targetOf(new PHPRegexTreeNodeBuilder(new PhpVersion(80400), PcreVersion::fromString('10.42'))), 'a pin below the bundled engine narrows'); - self::assertSame('10.44', $targetOf(new PHPRegexTreeNodeBuilder(new PhpVersion(80400), PcreVersion::fromString('10.44'))), 'a pin at the bundled engine judges it'); + self::assertSame('10.44', $targetOf(new PHPRegexReader(new PhpVersion(80400), $runtimeEngine)), 'a pin above the bundled engine is ignored'); + self::assertSame('10.42', $targetOf(new PHPRegexReader(new PhpVersion(80400), PcreVersion::fromString('10.42'))), 'a pin below the bundled engine narrows'); + self::assertSame('10.44', $targetOf(new PHPRegexReader(new PhpVersion(80400), PcreVersion::fromString('10.44'))), 'a pin at the bundled engine judges it'); } public function testEngineRefusedPatternsGetNoShapeAndBracketDelimitersWalk(): void