From ac90fc26b60b91cfc34d35919caa61a884f4370a Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 23 Apr 2023 21:00:38 +0200 Subject: [PATCH 1/3] BUGFIX: parenthesis in expressions previously `ExpressionNode::fromString("(foo)")` couldnt be parsed (it worked though when used in return statement of a module) but the real fix, is that you couldnt write `return ((foo))` as $tokens is reassigned but not passed by ref --- src/Parser/Ast/ExpressionNode.php | 11 +++--- src/Parser/Tokenizer/LookAhead.php | 4 +- src/Parser/Tokenizer/Scanner.php | 16 ++++++++ .../Identifier/IdentifierTranspilerTest.php | 38 ++++++++++++++++++- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/Parser/Ast/ExpressionNode.php b/src/Parser/Ast/ExpressionNode.php index 5c09d7a1..2f2538b7 100644 --- a/src/Parser/Ast/ExpressionNode.php +++ b/src/Parser/Ast/ExpressionNode.php @@ -39,10 +39,11 @@ private function __construct( public static function fromString(string $expressionAsString): self { + $tokens = Tokenizer::fromSource( + Source::fromString($expressionAsString) + )->getIterator(); return self::fromTokens( - Tokenizer::fromSource( - Source::fromString($expressionAsString) - )->getIterator() + $tokens ); } @@ -51,7 +52,7 @@ public static function fromString(string $expressionAsString): self * @param Precedence $precedence * @return self */ - public static function fromTokens(\Iterator $tokens, Precedence $precedence = Precedence::SEQUENCE): self + public static function fromTokens(\Iterator &$tokens, Precedence $precedence = Precedence::SEQUENCE): self { Scanner::skipSpaceAndComments($tokens); @@ -62,7 +63,7 @@ public static function fromTokens(\Iterator $tokens, Precedence $precedence = Pr $lookAhead->shift(); while (true) { - switch ($lookAhead->type()) { + switch (Scanner::isEnd($lookAhead->tokens) ?: $lookAhead->type()) { case TokenType::STRING: case TokenType::COLON: case TokenType::COMMA: diff --git a/src/Parser/Tokenizer/LookAhead.php b/src/Parser/Tokenizer/LookAhead.php index 3b9f3176..ce14aec3 100644 --- a/src/Parser/Tokenizer/LookAhead.php +++ b/src/Parser/Tokenizer/LookAhead.php @@ -58,7 +58,9 @@ public function getIterator(): \Iterator yield $token; } - yield from $this->tokens; + if (!Scanner::isEnd($this->tokens)) { + yield from $this->tokens; + } } public function shift(): void diff --git a/src/Parser/Tokenizer/Scanner.php b/src/Parser/Tokenizer/Scanner.php index c08b0c62..0257e9ac 100644 --- a/src/Parser/Tokenizer/Scanner.php +++ b/src/Parser/Tokenizer/Scanner.php @@ -165,4 +165,20 @@ public static function isEnd(\Iterator $tokens): bool { return !$tokens->valid(); } + + /** + * @param \Iterator $tokens + */ + public static function debugPrint(\Iterator $tokens): string + { + $tokensAsArray = []; + while ($tokens->valid()) { + $tokensAsArray[] = [ + "type" => $tokens->current()->type, + "value" => $tokens->current()->value + ]; + $tokens->next(); + } + return json_encode($tokensAsArray, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR); + } } diff --git a/test/Unit/Target/Php/Transpiler/Identifier/IdentifierTranspilerTest.php b/test/Unit/Target/Php/Transpiler/Identifier/IdentifierTranspilerTest.php index 3d23768c..850472d2 100644 --- a/test/Unit/Target/Php/Transpiler/Identifier/IdentifierTranspilerTest.php +++ b/test/Unit/Target/Php/Transpiler/Identifier/IdentifierTranspilerTest.php @@ -23,10 +23,13 @@ namespace PackageFactory\ComponentEngine\Test\Unit\Target\Php\Transpiler\Identifier; use PackageFactory\ComponentEngine\Parser\Ast\EnumDeclarationNode; +use PackageFactory\ComponentEngine\Parser\Ast\ExpressionNode; use PackageFactory\ComponentEngine\Parser\Ast\IdentifierNode; +use PackageFactory\ComponentEngine\Target\Php\Transpiler\Expression\ExpressionTranspiler; use PackageFactory\ComponentEngine\Test\Unit\TypeSystem\Scope\Fixtures\DummyScope; use PackageFactory\ComponentEngine\Target\Php\Transpiler\Identifier\IdentifierTranspiler; use PackageFactory\ComponentEngine\TypeSystem\Type\EnumType\EnumStaticType; +use PackageFactory\ComponentEngine\TypeSystem\Type\StringType\StringType; use PHPUnit\Framework\TestCase; final class IdentifierTranspilerTest extends TestCase @@ -80,4 +83,37 @@ public function transpilesIdentifierNodesReferringToEnums(): void $actualTranspilationResult ); } -} \ No newline at end of file + + + public function identifierInParenthesisExamples(): mixed + { + // @todo find a better place for these tests, as we actually test the ExpressionNode + return [ + '(foo)' => ['(foo)', '$this->foo'], + '((foo))' => ['((foo))', '$this->foo'], + '(((foo)))' => ['(((foo)))', '$this->foo'] + ]; + } + + /** + * @dataProvider identifierInParenthesisExamples + * @test + */ + public function identifierInParenthesis(string $expression, string $expectedTranspilationResult): void + { + $expressionTranspiler = new ExpressionTranspiler( + scope: new DummyScope([ + "foo" => StringType::get() + ]) + ); + + $actualTranspilationResult = $expressionTranspiler->transpile( + ExpressionNode::fromString($expression) + ); + + $this->assertEquals( + $expectedTranspilationResult, + $actualTranspilationResult + ); + } +} From e67801702efc1fc9043462e755476a18ea72a4ab Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 23 Apr 2023 21:04:11 +0200 Subject: [PATCH 2/3] TASK: Move Scanner::isEnd into `$lookAhead->type()` and make it nullable --- src/Parser/Ast/ExpressionNode.php | 2 +- src/Parser/Tokenizer/LookAhead.php | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Parser/Ast/ExpressionNode.php b/src/Parser/Ast/ExpressionNode.php index 2f2538b7..21d9ed5e 100644 --- a/src/Parser/Ast/ExpressionNode.php +++ b/src/Parser/Ast/ExpressionNode.php @@ -63,7 +63,7 @@ public static function fromTokens(\Iterator &$tokens, Precedence $precedence = P $lookAhead->shift(); while (true) { - switch (Scanner::isEnd($lookAhead->tokens) ?: $lookAhead->type()) { + switch ($lookAhead->type()) { case TokenType::STRING: case TokenType::COLON: case TokenType::COMMA: diff --git a/src/Parser/Tokenizer/LookAhead.php b/src/Parser/Tokenizer/LookAhead.php index ce14aec3..6dfb7252 100644 --- a/src/Parser/Tokenizer/LookAhead.php +++ b/src/Parser/Tokenizer/LookAhead.php @@ -70,8 +70,11 @@ public function shift(): void Scanner::skipOne($this->tokens); } - public function type(): TokenType + public function type(): ?TokenType { + if (Scanner::isEnd($this->tokens)) { + return null; + } return Scanner::type($this->tokens); } } From 734aa46bd3355a6c8208ef409167e31beb65ebca Mon Sep 17 00:00:00 2001 From: mhsdesign <85400359+mhsdesign@users.noreply.github.com> Date: Sun, 23 Apr 2023 22:11:53 +0200 Subject: [PATCH 3/3] TASK: Fix Tokenizer not tokenizing all tokens for example ExpressionNode::fromString('(true) ? 42 : "foo"') would previously only tokenize `(true)` due to the brace this is not an issue, when used in the return statement --- src/Parser/Ast/ExpressionNode.php | 2 +- src/Parser/Tokenizer/Scanner.php | 8 +++++++- src/Parser/Tokenizer/Tokenizer.php | 5 ++++- .../TernaryOperation/TernaryOperationTranspilerTest.php | 3 ++- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/Parser/Ast/ExpressionNode.php b/src/Parser/Ast/ExpressionNode.php index 21d9ed5e..5080911d 100644 --- a/src/Parser/Ast/ExpressionNode.php +++ b/src/Parser/Ast/ExpressionNode.php @@ -126,7 +126,7 @@ public static function fromTokens(\Iterator &$tokens, Precedence $precedence = P ); } - while ($tokens->valid()) { + while (!Scanner::isEnd($tokens)) { Scanner::skipSpaceAndComments($tokens); switch (Scanner::type($tokens)) { diff --git a/src/Parser/Tokenizer/Scanner.php b/src/Parser/Tokenizer/Scanner.php index 0257e9ac..5aa89761 100644 --- a/src/Parser/Tokenizer/Scanner.php +++ b/src/Parser/Tokenizer/Scanner.php @@ -169,8 +169,14 @@ public static function isEnd(\Iterator $tokens): bool /** * @param \Iterator $tokens */ - public static function debugPrint(\Iterator $tokens): string + public static function debugPrint(\Iterator &$tokens): string { + $tokens = (function(): \Generator { + throw new \Exception('Once debugged, $tokens is empty.'); + // @phpstan-ignore-next-line + yield; + })(); + $tokensAsArray = []; while ($tokens->valid()) { $tokensAsArray[] = [ diff --git a/src/Parser/Tokenizer/Tokenizer.php b/src/Parser/Tokenizer/Tokenizer.php index 0760410c..818c2f1c 100644 --- a/src/Parser/Tokenizer/Tokenizer.php +++ b/src/Parser/Tokenizer/Tokenizer.php @@ -44,7 +44,10 @@ public static function fromSource(Source $source): Tokenizer */ public function getIterator(): \Iterator { - yield from self::block($this->source->getIterator()); + $fragments = $this->source->getIterator(); + while ($fragments->valid()) { + yield from self::block($fragments); + } } /** diff --git a/test/Unit/Target/Php/Transpiler/TernaryOperation/TernaryOperationTranspilerTest.php b/test/Unit/Target/Php/Transpiler/TernaryOperation/TernaryOperationTranspilerTest.php index 1a44e3d3..9ca02e1a 100644 --- a/test/Unit/Target/Php/Transpiler/TernaryOperation/TernaryOperationTranspilerTest.php +++ b/test/Unit/Target/Php/Transpiler/TernaryOperation/TernaryOperationTranspilerTest.php @@ -37,6 +37,7 @@ public function ternaryOperationExamples(): array { return [ 'true ? 42 : "foo"' => ['true ? 42 : "foo"', '(true ? 42 : \'foo\')'], + '(true) ? 42 : "foo"' => ['(true) ? 42 : "foo"', '(true ? 42 : \'foo\')'], 'a ? 42 : "foo"' => ['a ? 42 : "foo"', '($this->a ? 42 : \'foo\')'], 'true ? b : "foo"' => ['true ? b : "foo"', '(true ? $this->b : \'foo\')'], 'true ? 42 : c' => ['true ? 42 : c', '(true ? 42 : $this->c)'], @@ -70,4 +71,4 @@ public function transpilesTernaryOperationNodes(string $ternaryOperationAsString $actualTranspilationResult ); } -} \ No newline at end of file +}