Merge pull request #370 from phpDocumentor/retry-parser-docblocs

Fix docblock tag descriptions

phpstan.neon

@@ -4,5 +4,9 @@ parameters:
         - '#Method phpDocumentor\\Reflection\\DocBlock\\StandardTagFactory::createTag\(\) should return phpDocumentor\\Reflection\\DocBlock\\Tag but returns mixed#'
         - "#Strict comparison using !== between array{'name', 'type'} and array{'name', 'type'} will always evaluate to false#"
         - '#Call to static method Webmozart\\Assert\\Assert::implementsInterface\(\) with class-string#'
+        - '#Class PHPStan\\PhpDocParser\\Lexer\\Lexer does not have a constructor and must be instantiated without any parameters\.#'
+        - '#Class PHPStan\\PhpDocParser\\Parser\\ConstExprParser constructor invoked with 3 parameters, 0\-1 required\.#'
+        - '#Class PHPStan\\PhpDocParser\\Parser\\PhpDocParser constructor invoked with 6 parameters, 2\-3 required\.#'
+        - '#Class PHPStan\\PhpDocParser\\Parser\\TypeParser constructor invoked with 3 parameters\, 0\-1 required\.#'
     paths:
         - src

psalm.xml

@@ -48,6 +48,12 @@
             </errorLevel>
         </NoInterfaceProperties>
 
+        <TooManyArguments>
+            <errorLevel type="info">
+                <file name="src/DocBlock/Tags/Factory/AbstractPHPStanFactory.php"/>
+            </errorLevel>
+        </TooManyArguments>
+
         <RedundantConditionGivenDocblockType>
             <errorLevel type="info">
                 <!-- Psalm manage to infer a more precise type than PHPStan. notNull assert is needed for PHPStan but

src/DocBlock/Tags/Factory/AbstractPHPStanFactory.php

@@ -23,7 +23,9 @@ use PHPStan\PhpDocParser\Parser\TokenIterator;
 use PHPStan\PhpDocParser\Parser\TypeParser;
 use RuntimeException;
 
+use function ltrim;
 use function property_exists;
+use function rtrim;
 
 /**
  * Factory class creating tags using phpstan's parser
@@ -42,18 +44,28 @@ class AbstractPHPStanFactory implements Factory
 
     public function __construct(PHPStanFactory ...$factories)
     {
-        $this->lexer = new Lexer();
+        $this->lexer = new Lexer(true);
-        $constParser = new ConstExprParser();
+        $constParser = new ConstExprParser(true, true, ['lines' => true, 'indexes' => true]);
-        $this->parser = new PhpDocParser(new TypeParser($constParser), $constParser);
+        $this->parser = new PhpDocParser(
+            new TypeParser($constParser, true, ['lines' => true, 'indexes' => true]),
+            $constParser,
+            true,
+            true,
+            ['lines' => true, 'indexes' => true],
+            true
+        );
         $this->factories = $factories;
     }
 
     public function create(string $tagLine, ?TypeContext $context = null): Tag
     {
-        $tokens = new TokenIterator($this->lexer->tokenize($tagLine));
+        $tokens = $this->tokenizeLine($tagLine);
         $ast = $this->parser->parseTag($tokens);
         if (property_exists($ast->value, 'description') === true) {
-            $ast->value->setAttribute('description', $ast->value->description . $tokens->joinUntil(Lexer::TOKEN_END));
+            $ast->value->setAttribute(
+                'description',
+                $ast->value->description . $tokens->joinUntil(Lexer::TOKEN_END)
+            );
         }
 
         if ($context === null) {
@@ -75,4 +87,36 @@ class AbstractPHPStanFactory implements Factory
             $ast->name
         );
     }
+
+    /**
+     * Solve the issue with the lexer not tokenizing the line correctly
+     *
+     * This method is a workaround for the lexer that includes newline tokens with spaces. For
+     * phpstan this isn't an issue, as it doesn't do a lot of things with the indentation of descriptions.
+     * But for us is important to keep the indentation of the descriptions, so we need to fix the lexer output.
+     */
+    private function tokenizeLine(string $tagLine): TokenIterator
+    {
+        $tokens = $this->lexer->tokenize($tagLine);
+        $fixed = [];
+        foreach ($tokens as $token) {
+            if (($token[1] === Lexer::TOKEN_PHPDOC_EOL) && rtrim($token[0], " \t") !== $token[0]) {
+                $fixed[] = [
+                    rtrim($token[Lexer::VALUE_OFFSET], " \t"),
+                    Lexer::TOKEN_PHPDOC_EOL,
+                    $token[2] ?? null,
+                ];
+                $fixed[] = [
+                    ltrim($token[Lexer::VALUE_OFFSET], "\n\r"),
+                    Lexer::TOKEN_HORIZONTAL_WS,
+                    ($token[2] ?? null) + 1,
+                ];
+                continue;
+            }
+
+            $fixed[] = $token;
+        }
+
+        return new TokenIterator($fixed);
+    }
 }

tests/integration/InterpretingDocBlocksTest.php

@@ -338,7 +338,7 @@ DOC;
                         false,
                         new Description(
                             '{' . "\n" .
-                            'Optional. Array or string of arguments for installing a package. Default empty array.' . "\n" .
+                            '    Optional. Array or string of arguments for installing a package. Default empty array.' . "\n" .
                             "\n" .
                             '    @type string $source                      Required path to the package source. Default empty.' . "\n" .
                             '    @type string $destination                 Required path to a folder to install the package in.' . "\n" .
@@ -364,4 +364,71 @@ DOC;
             $docblock
         );
     }
+
+    public function testIndentationIsKept(): void
+    {
+        $docComment = <<<DOC
+	/**
+	 * Registers the script module if no script module with that script module
+	 * identifier has already been registered.
+	 *
+	 * @since 6.5.0
+	 *
+	 * @param array             \$deps     {
+	 *                                        Optional. List of dependencies.
+	 *
+	 *                                        @type string|array ...$0 {
+	 *                                            An array of script module identifiers of the dependencies of this script
+	 *                                            module. The dependencies can be strings or arrays. If they are arrays,
+	 *                                            they need an `id` key with the script module identifier, and can contain
+	 *                                            an `import` key with either `static` or `dynamic`. By default,
+	 *                                            dependencies that don't contain an `import` key are considered static.
+	 *
+	 *                                            @type string \$id     The script module identifier.
+	 *                                            @type string \$import Optional. Import type. May be either `static` or
+	 *                                                                 `dynamic`. Defaults to `static`.
+	 *                                        }
+	 *                                    }
+	 */
+DOC;
+
+        $factory = DocBlockFactory::createInstance();
+        $docblock = $factory->create($docComment);
+
+        self::assertEquals(
+            new DocBlock(
+                'Registers the script module if no script module with that script module
+identifier has already been registered.',
+                new Description(
+                    ''
+                ),
+                [
+                    new Since('6.5.0', new Description('')),
+                    new Param(
+                        'deps',
+                        new Array_(new Mixed_()),
+                        false,
+                        new Description("{
+    Optional. List of dependencies.
+
+    @type string|array ...$0 {
+        An array of script module identifiers of the dependencies of this script
+        module. The dependencies can be strings or arrays. If they are arrays,
+        they need an `id` key with the script module identifier, and can contain
+        an `import` key with either `static` or `dynamic`. By default,
+        dependencies that don't contain an `import` key are considered static.
+
+        @type string \$id     The script module identifier.
+        @type string \$import Optional. Import type. May be either `static` or
+                             `dynamic`. Defaults to `static`.
+    }
+}"
+                        )
+                    ),
+                ],
+                new Context('\\')
+            ),
+            $docblock
+        );
+    }
 }