Skip to content

Commit 3ec6fb3

Browse files
theodorejbtheodorejb
committed
Add token constants
1 parent 912c963 commit 3ec6fb3

File tree

2 files changed

+91
-57
lines changed

2 files changed

+91
-57
lines changed

‎src/Lexer.php‎

Lines changed: 80 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,40 @@
1212
*/
1313
final class Lexer extends Phlexer
1414
{
15+
const T_BOOLEAN = 'BOOLEAN';
16+
const T_CLOSE = 'CLOSE';
17+
const T_CLOSE_ARRAY = 'CLOSE_ARRAY';
18+
const T_CLOSE_BLOCK_PARAMS = 'CLOSE_BLOCK_PARAMS';
19+
const T_CLOSE_RAW_BLOCK = 'CLOSE_RAW_BLOCK';
20+
const T_CLOSE_SEXPR = 'CLOSE_SEXPR';
21+
const T_CLOSE_UNESCAPED = 'CLOSE_UNESCAPED';
22+
const T_COMMENT = 'COMMENT';
23+
const T_CONTENT = 'CONTENT';
24+
const T_DATA = 'DATA';
25+
const T_END_RAW_BLOCK = 'END_RAW_BLOCK';
26+
const T_EOF = 'EOF';
27+
const T_EQUALS = 'EQUALS';
28+
const T_ID = 'ID';
29+
const T_INVALID = 'INVALID';
30+
const T_INVERSE = 'INVERSE';
31+
const T_NULL = 'NULL';
32+
const T_NUMBER = 'NUMBER';
33+
const T_OPEN = 'OPEN';
34+
const T_OPEN_BLOCK = 'OPEN_BLOCK';
35+
const T_OPEN_BLOCK_PARAMS = 'OPEN_BLOCK_PARAMS';
36+
const T_OPEN_ENDBLOCK = 'OPEN_ENDBLOCK';
37+
const T_OPEN_INVERSE = 'OPEN_INVERSE';
38+
const T_OPEN_INVERSE_CHAIN = 'OPEN_INVERSE_CHAIN';
39+
const T_OPEN_PARTIAL = 'OPEN_PARTIAL';
40+
const T_OPEN_PARTIAL_BLOCK = 'OPEN_PARTIAL_BLOCK';
41+
const T_OPEN_RAW_BLOCK = 'OPEN_RAW_BLOCK';
42+
const T_OPEN_SEXPR = 'OPEN_SEXPR';
43+
const T_OPEN_UNESCAPED = 'OPEN_UNESCAPED';
44+
const T_PRIVATE_SEP = 'PRIVATE_SEP';
45+
const T_SEP = 'SEP';
46+
const T_STRING = 'STRING';
47+
const T_UNDEFINED = 'UNDEFINED';
48+
1549
public function __construct()
1650
{
1751
$LEFT_STRIP = $RIGHT_STRIP = '~';
@@ -42,42 +76,42 @@ public function __construct()
4276
$this->pushState('mu');
4377
}
4478

45-
return $this->yytext !== '' ? 'CONTENT' : null;
79+
return $this->yytext !== '' ? self::T_CONTENT : null;
4680
}),
4781

48-
new Rule([], '[^\\x00]+', fn() => 'CONTENT'),
82+
new Rule([], '[^\\x00]+', fn() => self::T_CONTENT),
4983

5084
// marks CONTENT up to the next mustache or escaped mustache
5185
new Rule(['emu'], '[^\\x00]{2,}?(?={{|\\\\{{|\\\\\\\\{{|\\Z)', function () {
5286
$this->popState();
53-
return 'CONTENT';
87+
return self::T_CONTENT;
5488
}),
5589

5690
// nested raw block will create stacked 'raw' condition
5791
new Rule(['raw'], '{{{{(?=[^\\/])', function () {
5892
$this->pushState('raw');
59-
return 'CONTENT';
93+
return self::T_CONTENT;
6094
}),
6195

6296
new Rule(['raw'], '{{{{\\/' . $CTRL_INVERSE . '(?=[=}\\s\\/.])}}}}', function () {
6397
$this->popState();
6498

6599
if ($this->topState() === 'raw') {
66-
return 'CONTENT';
100+
return self::T_CONTENT;
67101
} else {
68102
$this->strip(5, 9);
69-
return 'END_RAW_BLOCK';
103+
return self::T_END_RAW_BLOCK;
70104
}
71105
}),
72-
new Rule(['raw'], '[^\\x00]+?(?={{{{)', fn() => 'CONTENT'),
106+
new Rule(['raw'], '[^\\x00]+?(?={{{{)', fn() => self::T_CONTENT),
73107

74108
new Rule(['com'], '[\\s\\S]*?--' . $RIGHT_STRIP . '?}}', function () {
75109
$this->popState();
76-
return 'COMMENT';
110+
return self::T_COMMENT;
77111
}),
78112

79-
new Rule(['mu'], '\\(', fn() => 'OPEN_SEXPR'),
80-
new Rule(['mu'], '\\)', fn() => 'CLOSE_SEXPR'),
113+
new Rule(['mu'], '\\(', fn() => self::T_OPEN_SEXPR),
114+
new Rule(['mu'], '\\)', fn() => self::T_CLOSE_SEXPR),
81115

82116
new Rule(['mu'], '\\[', function () {
83117
// Assuming yy.syntax.square === 'string'. OPEN_ARRAY option not handled
@@ -86,30 +120,30 @@ public function __construct()
86120
$this->pushState('escl');
87121
return null;
88122
}),
89-
new Rule(['mu'], ']', fn() => 'CLOSE_ARRAY'),
123+
new Rule(['mu'], ']', fn() => self::T_CLOSE_ARRAY),
90124

91-
new Rule(['mu'], '{{{{', fn() => 'OPEN_RAW_BLOCK'),
125+
new Rule(['mu'], '{{{{', fn() => self::T_OPEN_RAW_BLOCK),
92126
new Rule(['mu'], '}}}}', function () {
93127
$this->popState();
94128
$this->pushState('raw');
95-
return 'CLOSE_RAW_BLOCK';
129+
return self::T_CLOSE_RAW_BLOCK;
96130
}),
97-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?>', fn() => 'OPEN_PARTIAL'),
98-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?#>', fn() => 'OPEN_PARTIAL_BLOCK'),
99-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?#\\*?', fn() => 'OPEN_BLOCK'),
100-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\/', fn() => 'OPEN_ENDBLOCK'),
131+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?>', fn() => self::T_OPEN_PARTIAL),
132+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?#>', fn() => self::T_OPEN_PARTIAL_BLOCK),
133+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?#\\*?', fn() => self::T_OPEN_BLOCK),
134+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\/', fn() => self::T_OPEN_ENDBLOCK),
101135
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\^\\s*' . $RIGHT_STRIP . '?}}', function () {
102136
$this->popState();
103-
return 'INVERSE';
137+
return self::T_INVERSE;
104138
}),
105139
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\s*else\\s*' . $RIGHT_STRIP . '?}}', function () {
106140
$this->popState();
107-
return 'INVERSE';
141+
return self::T_INVERSE;
108142
}),
109-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\^', fn() => 'OPEN_INVERSE'),
110-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\s*else', fn() => 'OPEN_INVERSE_CHAIN'),
111-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?{', fn() => 'OPEN_UNESCAPED'),
112-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?&', fn() => 'OPEN'),
143+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\^', fn() => self::T_OPEN_INVERSE),
144+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\s*else', fn() => self::T_OPEN_INVERSE_CHAIN),
145+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?{', fn() => self::T_OPEN_UNESCAPED),
146+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?&', fn() => self::T_OPEN),
113147
new Rule(['mu'], '{{' . $LEFT_STRIP . '?!--', function () {
114148
$this->rewind(strlen($this->yytext));
115149
$this->popState();
@@ -118,56 +152,56 @@ public function __construct()
118152
}),
119153
new Rule(['mu'], '{{' . $LEFT_STRIP . '?![\\s\\S]*?}}', function () {
120154
$this->popState();
121-
return 'COMMENT';
155+
return self::T_COMMENT;
122156
}),
123-
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\*?', fn() => 'OPEN'),
157+
new Rule(['mu'], '{{' . $LEFT_STRIP . '?\\*?', fn() => self::T_OPEN),
124158

125-
new Rule(['mu'], '=', fn() => 'EQUALS'),
126-
new Rule(['mu'], '\\.\\.', fn() => 'ID'),
127-
new Rule(['mu'], '\\.(?=' . $LOOKAHEAD . ')', fn() => 'ID'),
128-
new Rule(['mu'], '\\.#', fn() => 'PRIVATE_SEP'),
129-
new Rule(['mu'], '[\\/.]', fn() => 'SEP'),
159+
new Rule(['mu'], '=', fn() => self::T_EQUALS),
160+
new Rule(['mu'], '\\.\\.', fn() => self::T_ID),
161+
new Rule(['mu'], '\\.(?=' . $LOOKAHEAD . ')', fn() => self::T_ID),
162+
new Rule(['mu'], '\\.#', fn() => self::T_PRIVATE_SEP),
163+
new Rule(['mu'], '[\\/.]', fn() => self::T_SEP),
130164
new Rule(['mu'], '\\s+', fn() => null), // ignore whitespace
131165
new Rule(['mu'], '}' . $RIGHT_STRIP . '?}}', function () {
132166
$this->popState();
133-
return 'CLOSE_UNESCAPED';
167+
return self::T_CLOSE_UNESCAPED;
134168
}),
135169
new Rule(['mu'], $RIGHT_STRIP . '?}}', function () {
136170
$this->popState();
137-
return 'CLOSE';
171+
return self::T_CLOSE;
138172
}),
139173
// double-quoted string
140174
new Rule(['mu'], '"(\\\\["]|[^"])*"', function () {
141175
$this->strip(1, 2);
142176
$this->replace('/\\\\"/', '"');
143-
return 'STRING';
177+
return self::T_STRING;
144178
}),
145179
// single quoted string
146180
new Rule(['mu'], "'(\\\\[']|[^'])*'", function () {
147181
$this->strip(1, 2);
148182
$this->replace("/\\\\'/", "'");
149-
return 'STRING';
183+
return self::T_STRING;
150184
}),
151-
new Rule(['mu'], '@', fn() => 'DATA'),
152-
new Rule(['mu'], 'true(?=' . $LITERAL_LOOKAHEAD . ')', fn() => 'BOOLEAN'),
153-
new Rule(['mu'], 'false(?=' . $LITERAL_LOOKAHEAD . ')', fn() => 'BOOLEAN'),
154-
new Rule(['mu'], 'undefined(?=' . $LITERAL_LOOKAHEAD . ')', fn() => 'UNDEFINED'),
155-
new Rule(['mu'], 'null(?=' . $LITERAL_LOOKAHEAD . ')', fn() => 'NULL'),
156-
new Rule(['mu'], '\\-?[0-9]+(?:\\.[0-9]+)?(?=' . $LITERAL_LOOKAHEAD . ')', fn() => 'NUMBER'),
157-
new Rule(['mu'], 'as\\s+\\|', fn() => 'OPEN_BLOCK_PARAMS'),
158-
new Rule(['mu'], '\\|', fn() => 'CLOSE_BLOCK_PARAMS'),
185+
new Rule(['mu'], '@', fn() => self::T_DATA),
186+
new Rule(['mu'], 'true(?=' . $LITERAL_LOOKAHEAD . ')', fn() => self::T_BOOLEAN),
187+
new Rule(['mu'], 'false(?=' . $LITERAL_LOOKAHEAD . ')', fn() => self::T_BOOLEAN),
188+
new Rule(['mu'], 'undefined(?=' . $LITERAL_LOOKAHEAD . ')', fn() => self::T_UNDEFINED),
189+
new Rule(['mu'], 'null(?=' . $LITERAL_LOOKAHEAD . ')', fn() => self::T_NULL),
190+
new Rule(['mu'], '\\-?[0-9]+(?:\\.[0-9]+)?(?=' . $LITERAL_LOOKAHEAD . ')', fn() => self::T_NUMBER),
191+
new Rule(['mu'], 'as\\s+\\|', fn() => self::T_OPEN_BLOCK_PARAMS),
192+
new Rule(['mu'], '\\|', fn() => self::T_CLOSE_BLOCK_PARAMS),
159193

160-
new Rule(['mu'], $ID, fn() => 'ID'),
194+
new Rule(['mu'], $ID, fn() => self::T_ID),
161195

162196
new Rule(['escl'], '\\[(\\\\\\]|[^\\]])*\\]', function () {
163197
$this->replace('/\\\\([\\\\\\]])/', '$1');
164198
$this->popState();
165-
return 'ID';
199+
return self::T_ID;
166200
}),
167201

168-
new Rule(['mu'], '.', fn() => 'INVALID'),
202+
new Rule(['mu'], '.', fn() => self::T_INVALID),
169203

170-
new Rule(['INITIAL', 'mu'], '\\Z', fn() => 'EOF'),
204+
new Rule(['INITIAL', 'mu'], '\\Z', fn() => self::T_EOF),
171205
]);
172206
}
173207

‎test/LexerTest.php‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,10 @@ public function testSpecs(array $spec): void
3838
{
3939
// fix invalid expectations
4040
if ($spec['it'] === 'does not time out in a mustache with a single } followed by EOF') {
41-
$spec['expected'][] = ['name' => 'INVALID', 'text' => '}'];
41+
$spec['expected'][] = ['name' => Lexer::T_INVALID, 'text' => '}'];
4242
} elseif ($spec['it'] === 'does not time out in a mustache when invalid ID characters are used') {
43-
$spec['expected'][] = ['name' => 'INVALID', 'text' => '&'];
44-
$spec['expected'][] = ['name' => 'CLOSE', 'text' => '}}'];
43+
$spec['expected'][] = ['name' => Lexer::T_INVALID, 'text' => '&'];
44+
$spec['expected'][] = ['name' => Lexer::T_CLOSE, 'text' => '}}'];
4545
}
4646

4747
$lexer = new Lexer();
@@ -59,14 +59,14 @@ public function testLineNumbers(): void
5959
{{lines}}
6060
_tpl;
6161
$expected = [
62-
new Token('CONTENT', "This\nis a ", 1),
63-
new Token('OPEN', '{{', 2),
64-
new Token('ID', 'template', 2),
65-
new Token('CLOSE', '}}', 2),
66-
new Token('CONTENT', "\nwith multiple\n", 3),
67-
new Token('OPEN', '{{', 4),
68-
new Token('ID', 'lines', 4),
69-
new Token('CLOSE', '}}', 4),
62+
new Token(Lexer::T_CONTENT, "This\nis a ", 1),
63+
new Token(Lexer::T_OPEN, '{{', 2),
64+
new Token(Lexer::T_ID, 'template', 2),
65+
new Token(Lexer::T_CLOSE, '}}', 2),
66+
new Token(Lexer::T_CONTENT, "\nwith multiple\n", 3),
67+
new Token(Lexer::T_OPEN, '{{', 4),
68+
new Token(Lexer::T_ID, 'lines', 4),
69+
new Token(Lexer::T_CLOSE, '}}', 4),
7070
];
7171
$this->assertEquals($expected, (new Lexer())->tokenize($template));
7272
}

0 commit comments

Comments
 (0)