Generate the parser from doc/grammars with phplrt

.gitattributes

@@ -17,3 +17,4 @@ Makefile export-ignore
 phpstan-baseline.neon export-ignore
 phpstan.neon export-ignore
 phpunit.xml export-ignore
+src/Parser/Generated/** linguist-generated=true

.github/workflows/build.yml

@@ -175,6 +175,11 @@ jobs:
       - name: "Install the grammar toolchain"
         run: "make grammars-install"
 
+      - name: "Check the generated parser is up to date"
+        run: |
+          make grammars-build
+          git diff --exit-code src/Parser/Generated
+
       - name: "Tests"
         run: "make tests"
 

Makefile

@@ -45,3 +45,9 @@ phpstan-generate-baseline:
 .PHONY: grammars-install
 grammars-install:
 	composer install --no-interaction --working-dir tools/phplrt
+
+# Writes the parser of src/Parser/Generated out of doc/grammars. Run it after
+# changing a grammar or a reducer of src/Parser/AstBuilder.php.
+.PHONY: grammars-build
+grammars-build:
+	php tools/phplrt/build.php

UPGRADING.md

@@ -1,3 +1,16 @@
+Upgrading from phpstan/phpdoc-parser 2.x to 3.0
+=================================
+
+### The parser is generated from the grammars
+
+`ConstExprParser`, `TypeParser` and `PhpDocParser` are no longer written by hand. The parser is generated from the grammars in [`doc/grammars`](doc/grammars) with [phplrt](https://github.com/phplrt/phplrt), and the AST is built by `PHPStan\PhpDocParser\Parser\AstBuilder`. The library now requires `ondrejmirtes/phplrt-runtime`.
+
+Their constructors and `parse()` methods (and `PhpDocParser::parseTag()`, `PhpDocParser::parseTagValue()` and `ConstExprParser::parseDoctrineString()`) are the same, and so is the AST they build.
+
+* Some details of a `ParserException` thrown for an invalid input may be different: the token the parser stopped at, and the token it says it expected there.
+* These public methods of `TypeParser`, which were not meant to be called from the outside, are removed: `parseGeneric()`, `parseTemplateTagValue()`, `isHtml()` and `enrichWithAttributes()`.
+* The `TypeParser` and `ConstExprParser` given to the constructors of `TypeParser` and `PhpDocParser` are not used anymore.
+
 Upgrading from phpstan/phpdoc-parser 1.x to 2.0
 =================================
 

composer.json

@@ -3,7 +3,8 @@
 	"description": "PHPDoc parser with support for nullable, intersection and generic types",
 	"license": "MIT",
 	"require": {
-		"php": "^7.4 || ^8.0"
+		"php": "^7.4 || ^8.0",
+		"ondrejmirtes/phplrt-runtime": "^4.2.4"
 	},
 	"require-dev": {
 		"doctrine/annotations": "^2.0",

doc/grammars/README.md

@@ -36,6 +36,21 @@ run are left in `temp/fuzzy` to be looked at.
 That is a great deal more of the language than a hand-written corpus covers, and
 it is what replaced the `abnfgen`-driven fuzzer this project used before.
 
+## The parser is generated from them
+
+`src/Parser/Generated/GeneratedParser.php` is the parser of `phpdoc.pp3`, and
+it is written by `tools/phplrt/build.php` — never edit it by hand:
+
+    make grammars-install                  # the toolchain, which asks for PHP 8.4
+    make grammars-build
+
+Every rule is a method reading it the way the phplrt runtime would read the
+compiled grammar, and building its value as it goes: the rules a node of the
+AST is built out of are the ones `src/Parser/AstBuilder.php` has a `reduce`
+method for, so a change of a grammar or of a reducer is followed by
+`make grammars-build`. The `Grammars` job of the CI checks that the parser in
+the repository is the one the grammars write.
+
 ## What keeps them in step
 
 The fuzzer walks the grammars one way; `tests/PHPStan/Parser/GrammarSyncTest.php`
@@ -110,16 +125,19 @@ most of them:
 - **A rule reads exactly the tokens its method reads**, down to the line breaks
   around it.
 
-### The tokens are not read by phplrt
+### The lexer is written out of the `%token` declarations
 
-A grammar of this directory is not read by the lexer it declares: it is read by
-the very tokens `PHPStan\PhpDocParser\Lexer\Lexer` produces, handed over by
-`tools/phplrt/Fuzzer/TokenStream.php`.
+`tools/phplrt/build.php` writes the pattern of `PHPStan\PhpDocParser\Lexer\Lexer`
+out of the `%token` declarations of `lexemes.pp3`, in the order they are
+declared, into `GeneratedParser::LEXER_PATTERN`. Each token is marked by the
+identifier the grammar gives it, and `GeneratedParser::LEXER_TYPES` maps it back
+to the `Lexer::TOKEN_*` type `tokenize()` returns, so the tokens are the very
+ones they always were.
 
-The `%token` declarations therefore name the tokens and document the language
-without being what reads it. Some of them describe something the lexer never
-reads as a token of its own, and `TokenStream` is what tells those apart in the
-stream:
+Some declarations describe something the lexer never reads as a token of its
+own. They are left out of the pattern, and `src/Parser/Phplrt/TokenStream.php`
+tells them apart in the stream the parser reads, while the PHPDoc is being
+lexed:
 
 - a word the parser compares by value (`is`, `array`, `covariant`, `static`, …)
   — every one of them is still an ordinary name as well, which is why they are

doc/grammars/const-expr.pp3

@@ -88,8 +88,7 @@ ConstantArrayItem
  *  - BAR
  */
 ConstantFetch
-  : !ArrayKeyword() Identifier() <T_DOUBLE_COLON> ClassConstantName()
-  | !ArrayKeyword() Identifier()
+  : !ArrayKeyword() Identifier() ( <T_DOUBLE_COLON> ClassConstantName() )?
   ;
 
 /**

doc/grammars/lexemes.pp3

@@ -3,23 +3,25 @@
  *  The Lexemes Of The PHPDoc Language
  * -----------------------------------------------------------------------------
  *
- *  Every token the PHPDoc language is read into, written the way
- *  "PHPStan\PhpDocParser\Lexer\Lexer" reads it.
+ *  Every token the PHPDoc language is read into.
  *
- *  A grammar of this directory is not read by the lexer it declares: it is read
- *  by the very tokens "Lexer" produces, handed over by
- *  "tools/phplrt/Fuzzer/TokenStream.php". The declarations below therefore
- *  serve three purposes:
+ *  "PHPStan\PhpDocParser\Lexer\Lexer" is written out of these declarations by
+ *  "tools/phplrt/build.php": every token that is not classified below becomes
+ *  an alternative of its pattern, tried in the order written here, so the
+ *  order is the one that decides between two tokens matching at one place.
+ *  The declarations also serve the grammar-driven fuzzer, which writes its
+ *  inputs out of them.
  *
- *   - they name the tokens the rules are written in terms of;
- *   - they document the language down to its lexical level;
- *   - they are what the grammar-driven fuzzer writes its inputs from.
+ *  Every group of a pattern is non-capturing: "preg_match_all()" collects
+ *  every capturing group for every token the lexer reads.
  *
- *  A block of tokens marked "classified" is not read as a token of its own by
- *  the lexer. Such a token is a T_IDENTIFIER (or a bracket, or an asterisk)
- *  whose surroundings or whose very value the hand-written parser asks about,
- *  and telling those apart in the token stream is what lets a grammar written
- *  without semantic predicates say the same thing.
+ *  A classified token is not an alternative of the pattern: the keywords, the
+ *  tags read by a rule of their own, the tokens ending in "_WS" and the "<" of
+ *  an HTML tag. Such a token is a T_IDENTIFIER (or a tag, a bracket, or an
+ *  asterisk) whose surroundings or whose very value the grammar asks about.
+ *  The lexer reads it as that token, and tells it apart only in the stream it
+ *  hands to the parser ("src/Parser/Phplrt/TokenStream.php"), which is what
+ *  lets a grammar written without semantic predicates say the same thing.
  */
 
 %pragma lexer.pcre.disable u
@@ -79,8 +81,8 @@
 //  Literals and names
 // -----------------------------------------------------------------------------
 
-%token T_FLOAT                       [+\-]?(?:(?:[0-9]++(_[0-9]++)*\.[0-9]*+(_[0-9]++)*(?:e[+\-]?[0-9]++(_[0-9]++)*)?)|(?:[0-9]*+(_[0-9]++)*\.[0-9]++(_[0-9]++)*(?:e[+\-]?[0-9]++(_[0-9]++)*)?)|(?:[0-9]++(_[0-9]++)*e[+\-]?[0-9]++(_[0-9]++)*))
-%token T_INTEGER                     [+\-]?(?:(?:0b[0-1]++(_[0-1]++)*)|(?:0o[0-7]++(_[0-7]++)*)|(?:0x[0-9a-f]++(_[0-9a-f]++)*)|(?:[0-9]++(_[0-9]++)*))
+%token T_FLOAT                       [+\-]?(?:(?:[0-9]++(?:_[0-9]++)*\.[0-9]*+(?:_[0-9]++)*(?:e[+\-]?[0-9]++(?:_[0-9]++)*)?)|(?:[0-9]*+(?:_[0-9]++)*\.[0-9]++(?:_[0-9]++)*(?:e[+\-]?[0-9]++(?:_[0-9]++)*)?)|(?:[0-9]++(?:_[0-9]++)*e[+\-]?[0-9]++(?:_[0-9]++)*))
+%token T_INTEGER                     [+\-]?(?:(?:0b[0-1]++(?:_[0-1]++)*)|(?:0o[0-7]++(?:_[0-7]++)*)|(?:0x[0-9a-f]++(?:_[0-9a-f]++)*)|(?:[0-9]++(?:_[0-9]++)*))
 %token T_SINGLE_QUOTED_STRING        '(?:\\[^\r\n]|[^'\r\n\\])*+'
 %token T_DOUBLE_QUOTED_STRING        "(?:\\[^\r\n]|[^"\r\n\\])*+"
 %token T_DOCTRINE_ANNOTATION_STRING  "(?:""|[^"])*+"
@@ -120,14 +122,14 @@
 %token T_ARROW                       ->
 %token T_EQUAL                       =
 %token T_COLON                       :
-%token T_WILDCARD                    \*
 
 // -----------------------------------------------------------------------------
 //  PHPDoc structure
 // -----------------------------------------------------------------------------
 
 %token T_OPEN_PHPDOC                 /\*\*(?=\s)\x20?+
 %token T_CLOSE_PHPDOC                \*/
+%token T_WILDCARD                    \*
 
 /**
  * The tags whose value is read by a rule of its own, told apart from one

doc/grammars/phpdoc-block.pp3

@@ -26,13 +26,18 @@ PhpDoc
   : <T_OPEN_PHPDOC> <T_PHPDOC_EOL>? ( !<T_CLOSE_PHPDOC> PhpDocChildren() )? <T_CLOSE_PHPDOC>
   ;
 
+/**
+ * The children one after another, written as a repetition rather than as a
+ * rule calling itself once per child, so that a PHPDoc of many tags is read
+ * without nesting a rule per tag.
+ */
 PhpDocChildren
-  : UnreadTagChild() <T_PHPDOC_EOL>? MorePhpDocChildren()?
-  | PhpDocChild() ( <T_PHPDOC_EOL> MorePhpDocChildren()? )?
+  : PhpDocChildrenItem() ( !<T_CLOSE_PHPDOC> PhpDocChildrenItem() )*
   ;
 
-MorePhpDocChildren
-  : !<T_CLOSE_PHPDOC> PhpDocChildren()
+PhpDocChildrenItem
+  : UnreadTagChild() <T_PHPDOC_EOL>?
+  | PhpDocChild() ( <T_PHPDOC_EOL> | &<T_CLOSE_PHPDOC> )
   ;
 
 /**
@@ -166,8 +171,7 @@ ParameterNameStart
  * space, which is what tells "@var Foo the one" from "@var Foo|the one".
  */
 VarTagValue
-  : Type() VariableName() Description()
-  | Type() DescriptionAfterType()
+  : Type() ( VariableName() Description() | DescriptionAfterType() )
   ;
 
 VariableName
@@ -189,10 +193,13 @@ VariableName
  *  meant.
  */
 MethodTagValue
-  : StaticType() Type() Identifier() MethodTagValueTail()
-  | StaticType() Type() MethodTagValueTail()
-  | Type() Identifier() MethodTagValueTail()
-  | Type() MethodTagValueTail()
+  : StaticType() Type() MethodNameAndTail()
+  | Type() MethodNameAndTail()
+  ;
+
+MethodNameAndTail
+  : Identifier() MethodTagValueTail()
+  | MethodTagValueTail()
   ;
 
 /**
@@ -354,12 +361,15 @@ DoctrineDescription
  * to it, while one written after it does not.
  */
 Text
-  : TextLine() TextTail()?
+  : TextLine() TextTail()*
   ;
 
+/**
+ * A line with something on it, along with the blank lines before it, so that
+ * the blank lines written after the last such line are left unread.
+ */
 TextTail
-  : <T_PHPDOC_EOL> !TextStop() TextLineWithSomethingOnIt() TextTail()?
-  | <T_PHPDOC_EOL> !TextStop() TextTail()
+  : ( <T_PHPDOC_EOL> !TextStop() )+ TextLineWithSomethingOnIt()
   ;
 
 TextLine
@@ -579,8 +589,7 @@ NonTagTextToken
  *  predicate says.
  */
 DoctrineTag
-  : DoctrineTagToken() DoctrineArguments() DoctrineDescription()
-  | DoctrineTagToken() !ParenthesesOpen() DoctrineDescription()
+  : DoctrineTagToken() ( DoctrineArguments() | !ParenthesesOpen() ) DoctrineDescription()
   | GenericTagToken() DoctrineArguments() DoctrineDescription()
   ;
 

phpcs.xml

@@ -8,6 +8,9 @@
 	<arg name="cache" value="tmp/phpcs"/>
 	<arg value="sp"/>
 
+	<!-- written by tools/phplrt/build.php -->
+	<exclude-pattern>src/Parser/Generated/*</exclude-pattern>
+
 	<rule ref="build-cs/phpcs.xml">
 		<exclude name="Squiz.Strings.DoubleQuoteUsage.ContainsVar"/>
 	</rule>

phpstan.neon

@@ -6,8 +6,11 @@ parameters:
 		- src
 		- tests
 	excludePaths:
-		- tests/PHPStan/*/data/*
-		- tests/PHPStan/Parser/Doctrine/ApiResource.php
+		analyse:
+			- src/Parser/Generated/*
+		analyseAndScan:
+			- tests/PHPStan/*/data/*
+			- tests/PHPStan/Parser/Doctrine/ApiResource.php
 	level: 8
 	ignoreErrors:
 		- '#^Dynamic call to static method PHPUnit\\Framework\\Assert#'