Fix docblock tag descriptions

Newlines and whitespaces were not handled as before. This caused
issues for some users because our indent-recuction was broken.
The cause seems to be an upstream issue in phpstan parser which
is not resolved yet. But this work around post processing the tokens
helps us to make it work as before.

phpstan.neon

@@ -4,5 +4,9 @@ parameters:
         - '#Method phpDocumentor\\Reflection\\DocBlock\\StandardTagFactory::createTag\(\) should return phpDocumentor\\Reflection\\DocBlock\\Tag but returns mixed#'
         - "#Strict comparison using !== between array{'name', 'type'} and array{'name', 'type'} will always evaluate to false#"
         - '#Call to static method Webmozart\\Assert\\Assert::implementsInterface\(\) with class-string#'
+        - '#Class PHPStan\\PhpDocParser\\Lexer\\Lexer does not have a constructor and must be instantiated without any parameters\.#'
+        - '#Class PHPStan\\PhpDocParser\\Parser\\ConstExprParser constructor invoked with 3 parameters, 0\-1 required\.#'
+        - '#Class PHPStan\\PhpDocParser\\Parser\\PhpDocParser constructor invoked with 6 parameters, 2\-3 required\.#'
+        - '#Class PHPStan\\PhpDocParser\\Parser\\TypeParser constructor invoked with 3 parameters\, 0\-1 required\.#'
     paths:
         - src

psalm.xml

@@ -48,6 +48,12 @@
             </errorLevel>
         </NoInterfaceProperties>
 
+        <TooManyArguments>
+            <errorLevel type="info">
+                <file name="src/DocBlock/Tags/Factory/AbstractPHPStanFactory.php"/>
+            </errorLevel>
+        </TooManyArguments>
+
         <RedundantConditionGivenDocblockType>
             <errorLevel type="info">
                 <!-- Psalm manage to infer a more precise type than PHPStan. notNull assert is needed for PHPStan but

src/DocBlock/Tags/Factory/AbstractPHPStanFactory.php

@@ -23,7 +23,9 @@ use PHPStan\PhpDocParser\Parser\TokenIterator;
 use PHPStan\PhpDocParser\Parser\TypeParser;
 use RuntimeException;
 
+use function ltrim;
 use function property_exists;
+use function rtrim;
 
 /**
  * Factory class creating tags using phpstan's parser
@@ -42,18 +44,28 @@ class AbstractPHPStanFactory implements Factory
 
     public function __construct(PHPStanFactory ...$factories)
     {
-        $this->lexer = new Lexer();
-        $constParser = new ConstExprParser();
-        $this->parser = new PhpDocParser(new TypeParser($constParser), $constParser);
+        $this->lexer = new Lexer(true);
+        $constParser = new ConstExprParser(true, true, ['lines' => true, 'indexes' => true]);
+        $this->parser = new PhpDocParser(
+            new TypeParser($constParser, true, ['lines' => true, 'indexes' => true]),
+            $constParser,
+            true,
+            true,
+            ['lines' => true, 'indexes' => true],
+            true
+        );
         $this->factories = $factories;
     }
 
     public function create(string $tagLine, ?TypeContext $context = null): Tag
     {
-        $tokens = new TokenIterator($this->lexer->tokenize($tagLine));
+        $tokens = $this->tokenizeLine($tagLine);
         $ast = $this->parser->parseTag($tokens);
         if (property_exists($ast->value, 'description') === true) {
-            $ast->value->setAttribute('description', $ast->value->description . $tokens->joinUntil(Lexer::TOKEN_END));
+            $ast->value->setAttribute(
+                'description',
+                $ast->value->description . $tokens->joinUntil(Lexer::TOKEN_END)
+            );
         }
 
         if ($context === null) {
@@ -75,4 +87,36 @@ class AbstractPHPStanFactory implements Factory
             $ast->name
         );
     }
+
+    /**
+     * Solve the issue with the lexer not tokenizing the line correctly
+     *
+     * This method is a workaround for the lexer that includes newline tokens with spaces. For
+     * phpstan this isn't an issue, as it doesn't do a lot of things with the indentation of descriptions.
+     * But for us is important to keep the indentation of the descriptions, so we need to fix the lexer output.
+     */
+    private function tokenizeLine(string $tagLine): TokenIterator
+    {
+        $tokens = $this->lexer->tokenize($tagLine);
+        $fixed = [];
+        foreach ($tokens as $token) {
+            if (($token[1] === Lexer::TOKEN_PHPDOC_EOL) && rtrim($token[0], " \t") !== $token[0]) {
+                $fixed[] = [
+                    rtrim($token[Lexer::VALUE_OFFSET], " \t"),
+                    Lexer::TOKEN_PHPDOC_EOL,
+                    $token[2] ?? null,
+                ];
+                $fixed[] = [
+                    ltrim($token[Lexer::VALUE_OFFSET], "\n\r"),
+                    Lexer::TOKEN_HORIZONTAL_WS,
+                    ($token[2] ?? null) + 1,
+                ];
+                continue;
+            }
+
+            $fixed[] = $token;
+        }
+
+        return new TokenIterator($fixed);
+    }
 }

tests/integration/InterpretingDocBlocksTest.php

@@ -338,7 +338,7 @@ DOC;
                         false,
                         new Description(
                             '{' . "\n" .
-                            'Optional. Array or string of arguments for installing a package. Default empty array.' . "\n" .
+                            '    Optional. Array or string of arguments for installing a package. Default empty array.' . "\n" .
                             "\n" .
                             '    @type string $source                      Required path to the package source. Default empty.' . "\n" .
                             '    @type string $destination                 Required path to a folder to install the package in.' . "\n" .
@@ -364,4 +364,71 @@ DOC;
             $docblock
         );
     }
+
+    public function testIndentationIsKept(): void
+    {
+        $docComment = <<<DOC
+	/**
+	 * Registers the script module if no script module with that script module
+	 * identifier has already been registered.
+	 *
+	 * @since 6.5.0
+	 *
+	 * @param array             \$deps     {
+	 *                                        Optional. List of dependencies.
+	 *
+	 *                                        @type string|array ...$0 {
+	 *                                            An array of script module identifiers of the dependencies of this script
+	 *                                            module. The dependencies can be strings or arrays. If they are arrays,
+	 *                                            they need an `id` key with the script module identifier, and can contain
+	 *                                            an `import` key with either `static` or `dynamic`. By default,
+	 *                                            dependencies that don't contain an `import` key are considered static.
+	 *
+	 *                                            @type string \$id     The script module identifier.
+	 *                                            @type string \$import Optional. Import type. May be either `static` or
+	 *                                                                 `dynamic`. Defaults to `static`.
+	 *                                        }
+	 *                                    }
+	 */
+DOC;
+
+        $factory = DocBlockFactory::createInstance();
+        $docblock = $factory->create($docComment);
+
+        self::assertEquals(
+            new DocBlock(
+                'Registers the script module if no script module with that script module
+identifier has already been registered.',
+                new Description(
+                    ''
+                ),
+                [
+                    new Since('6.5.0', new Description('')),
+                    new Param(
+                        'deps',
+                        new Array_(new Mixed_()),
+                        false,
+                        new Description("{
+    Optional. List of dependencies.
+
+    @type string|array ...$0 {
+        An array of script module identifiers of the dependencies of this script
+        module. The dependencies can be strings or arrays. If they are arrays,
+        they need an `id` key with the script module identifier, and can contain
+        an `import` key with either `static` or `dynamic`. By default,
+        dependencies that don't contain an `import` key are considered static.
+
+        @type string \$id     The script module identifier.
+        @type string \$import Optional. Import type. May be either `static` or
+                             `dynamic`. Defaults to `static`.
+    }
+}"
+                        )
+                    ),
+                ],
+                new Context('\\')
+            ),
+            $docblock
+        );
+    }
 }