This repository was archived by the owner on Jul 11, 2025. It is now read-only.
forked from phoboslab/JavaScriptCore-iOS
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathLexer.h
More file actions
393 lines (331 loc) · 12.5 KB
/
Lexer.h
File metadata and controls
393 lines (331 loc) · 12.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
/*
* Copyright (C) 1999-2000 Harri Porten ([email protected])
* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2011, 2012, 2013 Apple Inc. All rights reserved.
* Copyright (C) 2010 Zoltan Herczeg ([email protected])
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public License
* along with this library; see the file COPYING.LIB. If not, write to
* the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
#ifndef Lexer_h
#define Lexer_h
#include "Lookup.h"
#include "ParserArena.h"
#include "ParserTokens.h"
#include "SourceCode.h"
#include <wtf/ASCIICType.h>
#include <wtf/SegmentedVector.h>
#include <wtf/Vector.h>
#include <wtf/unicode/Unicode.h>
namespace JSC {
class Keywords {
public:
bool isKeyword(const Identifier& ident) const
{
return m_keywordTable.entry(m_vm, ident);
}
const HashEntry* getKeyword(const Identifier& ident) const
{
return m_keywordTable.entry(m_vm, ident);
}
~Keywords()
{
m_keywordTable.deleteTable();
}
private:
friend class VM;
explicit Keywords(VM&);
VM& m_vm;
const HashTable m_keywordTable;
};
enum LexerFlags {
LexerFlagsIgnoreReservedWords = 1,
LexerFlagsDontBuildStrings = 2,
LexexFlagsDontBuildKeywords = 4
};
template <typename T>
class Lexer {
WTF_MAKE_NONCOPYABLE(Lexer);
WTF_MAKE_FAST_ALLOCATED;
public:
Lexer(VM*);
~Lexer();
// Character manipulation functions.
static bool isWhiteSpace(T character);
static bool isLineTerminator(T character);
static unsigned char convertHex(int c1, int c2);
static UChar convertUnicode(int c1, int c2, int c3, int c4);
// Functions to set up parsing.
void setCode(const SourceCode&, ParserArena*);
void setIsReparsing() { m_isReparsing = true; }
bool isReparsing() const { return m_isReparsing; }
JSTokenType lex(JSToken*, unsigned, bool strictMode);
bool nextTokenIsColon();
int lineNumber() const { return m_lineNumber; }
ALWAYS_INLINE int currentOffset() const { return offsetFromSourcePtr(m_code); }
ALWAYS_INLINE int currentLineStartOffset() const { return offsetFromSourcePtr(m_lineStart); }
ALWAYS_INLINE JSTextPosition currentPosition() const
{
return JSTextPosition(m_lineNumber, currentOffset(), currentLineStartOffset());
}
JSTextPosition positionBeforeLastNewline() const { return m_positionBeforeLastNewline; }
void setLastLineNumber(int lastLineNumber) { m_lastLineNumber = lastLineNumber; }
int lastLineNumber() const { return m_lastLineNumber; }
bool prevTerminator() const { return m_terminator; }
bool scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix = 0);
bool skipRegExp();
// Functions for use after parsing.
bool sawError() const { return m_error; }
String getErrorMessage() const { return m_lexErrorMessage; }
void clear();
void setOffset(int offset, int lineStartOffset)
{
m_error = 0;
m_lexErrorMessage = String();
m_code = sourcePtrFromOffset(offset);
m_lineStart = sourcePtrFromOffset(lineStartOffset);
ASSERT(currentOffset() >= currentLineStartOffset());
m_buffer8.resize(0);
m_buffer16.resize(0);
if (LIKELY(m_code < m_codeEnd))
m_current = *m_code;
else
m_current = 0;
}
void setLineNumber(int line)
{
m_lineNumber = line;
}
SourceProvider* sourceProvider() const { return m_source->provider(); }
JSTokenType lexExpectIdentifier(JSToken*, unsigned, bool strictMode);
private:
void record8(int);
void append8(const T*, size_t);
void record16(int);
void record16(T);
void append16(const LChar*, size_t);
void append16(const UChar* characters, size_t length) { m_buffer16.append(characters, length); }
ALWAYS_INLINE void shift();
ALWAYS_INLINE bool atEnd() const;
ALWAYS_INLINE T peek(int offset) const;
struct UnicodeHexValue {
enum ValueType { ValidHex, IncompleteHex, InvalidHex };
explicit UnicodeHexValue(int value)
: m_value(value)
{
}
explicit UnicodeHexValue(ValueType type)
: m_value(type == IncompleteHex ? -2 : -1)
{
}
ValueType valueType() const
{
if (m_value >= 0)
return ValidHex;
return m_value == -2 ? IncompleteHex : InvalidHex;
}
bool isValid() const { return m_value >= 0; }
int value() const
{
ASSERT(m_value >= 0);
return m_value;
}
private:
int m_value;
};
UnicodeHexValue parseFourDigitUnicodeHex();
void shiftLineTerminator();
ALWAYS_INLINE int offsetFromSourcePtr(const T* ptr) const { return ptr - m_codeStart; }
ALWAYS_INLINE const T* sourcePtrFromOffset(int offset) const { return m_codeStart + offset; }
String invalidCharacterMessage() const;
ALWAYS_INLINE const T* currentSourcePtr() const;
ALWAYS_INLINE void setOffsetFromSourcePtr(const T* sourcePtr, unsigned lineStartOffset) { setOffset(offsetFromSourcePtr(sourcePtr), lineStartOffset); }
ALWAYS_INLINE void setCodeStart(const StringImpl*);
ALWAYS_INLINE const Identifier* makeIdentifier(const LChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeIdentifier(const UChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeLCharIdentifier(const LChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeLCharIdentifier(const UChar* characters, size_t length);
ALWAYS_INLINE const Identifier* makeRightSizedIdentifier(const UChar* characters, size_t length, UChar orAllChars);
ALWAYS_INLINE const Identifier* makeIdentifierLCharFromUChar(const UChar* characters, size_t length);
ALWAYS_INLINE bool lastTokenWasRestrKeyword() const;
template <int shiftAmount> void internalShift();
template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType parseKeyword(JSTokenData*);
template <bool shouldBuildIdentifiers> ALWAYS_INLINE JSTokenType parseIdentifier(JSTokenData*, unsigned lexerFlags, bool strictMode);
template <bool shouldBuildIdentifiers> NEVER_INLINE JSTokenType parseIdentifierSlowCase(JSTokenData*, unsigned lexerFlags, bool strictMode);
enum StringParseResult {
StringParsedSuccessfully,
StringUnterminated,
StringCannotBeParsed
};
template <bool shouldBuildStrings> ALWAYS_INLINE StringParseResult parseString(JSTokenData*, bool strictMode);
template <bool shouldBuildStrings> NEVER_INLINE StringParseResult parseStringSlowCase(JSTokenData*, bool strictMode);
ALWAYS_INLINE void parseHex(double& returnValue);
ALWAYS_INLINE bool parseOctal(double& returnValue);
ALWAYS_INLINE bool parseDecimal(double& returnValue);
ALWAYS_INLINE void parseNumberAfterDecimalPoint();
ALWAYS_INLINE bool parseNumberAfterExponentIndicator();
ALWAYS_INLINE bool parseMultilineComment();
static const size_t initialReadBufferCapacity = 32;
int m_lineNumber;
int m_lastLineNumber;
Vector<LChar> m_buffer8;
Vector<UChar> m_buffer16;
bool m_terminator;
int m_lastToken;
const SourceCode* m_source;
unsigned m_sourceOffset;
const T* m_code;
const T* m_codeStart;
const T* m_codeEnd;
const T* m_codeStartPlusOffset;
const T* m_lineStart;
JSTextPosition m_positionBeforeLastNewline;
bool m_isReparsing;
bool m_atLineStart;
bool m_error;
String m_lexErrorMessage;
T m_current;
IdentifierArena* m_arena;
VM* m_vm;
};
template <>
ALWAYS_INLINE bool Lexer<LChar>::isWhiteSpace(LChar ch)
{
return ch == ' ' || ch == '\t' || ch == 0xB || ch == 0xC || ch == 0xA0;
}
template <>
ALWAYS_INLINE bool Lexer<UChar>::isWhiteSpace(UChar ch)
{
return (ch < 256) ? Lexer<LChar>::isWhiteSpace(static_cast<LChar>(ch)) : (u_charType(ch) == U_SPACE_SEPARATOR || ch == 0xFEFF);
}
template <>
ALWAYS_INLINE bool Lexer<LChar>::isLineTerminator(LChar ch)
{
return ch == '\r' || ch == '\n';
}
template <>
ALWAYS_INLINE bool Lexer<UChar>::isLineTerminator(UChar ch)
{
return ch == '\r' || ch == '\n' || (ch & ~1) == 0x2028;
}
template <typename T>
inline unsigned char Lexer<T>::convertHex(int c1, int c2)
{
return (toASCIIHexValue(c1) << 4) | toASCIIHexValue(c2);
}
template <typename T>
inline UChar Lexer<T>::convertUnicode(int c1, int c2, int c3, int c4)
{
return (convertHex(c1, c2) << 8) | convertHex(c3, c4);
}
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeIdentifier(const LChar* characters, size_t length)
{
return &m_arena->makeIdentifier(m_vm, characters, length);
}
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeIdentifier(const UChar* characters, size_t length)
{
return &m_arena->makeIdentifier(m_vm, characters, length);
}
template <>
ALWAYS_INLINE const Identifier* Lexer<LChar>::makeRightSizedIdentifier(const UChar* characters, size_t length, UChar)
{
return &m_arena->makeIdentifierLCharFromUChar(m_vm, characters, length);
}
template <>
ALWAYS_INLINE const Identifier* Lexer<UChar>::makeRightSizedIdentifier(const UChar* characters, size_t length, UChar orAllChars)
{
if (!(orAllChars & ~0xff))
return &m_arena->makeIdentifierLCharFromUChar(m_vm, characters, length);
return &m_arena->makeIdentifier(m_vm, characters, length);
}
template <>
ALWAYS_INLINE void Lexer<LChar>::setCodeStart(const StringImpl* sourceString)
{
ASSERT(sourceString->is8Bit());
m_codeStart = sourceString->characters8();
}
template <>
ALWAYS_INLINE void Lexer<UChar>::setCodeStart(const StringImpl* sourceString)
{
ASSERT(!sourceString->is8Bit());
m_codeStart = sourceString->characters16();
}
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeIdentifierLCharFromUChar(const UChar* characters, size_t length)
{
return &m_arena->makeIdentifierLCharFromUChar(m_vm, characters, length);
}
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeLCharIdentifier(const LChar* characters, size_t length)
{
return &m_arena->makeIdentifier(m_vm, characters, length);
}
template <typename T>
ALWAYS_INLINE const Identifier* Lexer<T>::makeLCharIdentifier(const UChar* characters, size_t length)
{
return &m_arena->makeIdentifierLCharFromUChar(m_vm, characters, length);
}
template <typename T>
ALWAYS_INLINE JSTokenType Lexer<T>::lexExpectIdentifier(JSToken* tokenRecord, unsigned lexerFlags, bool strictMode)
{
JSTokenData* tokenData = &tokenRecord->m_data;
JSTokenLocation* tokenLocation = &tokenRecord->m_location;
ASSERT((lexerFlags & LexerFlagsIgnoreReservedWords));
const T* start = m_code;
const T* ptr = start;
const T* end = m_codeEnd;
JSTextPosition startPosition = currentPosition();
if (ptr >= end) {
ASSERT(ptr == end);
goto slowCase;
}
if (!WTF::isASCIIAlpha(*ptr))
goto slowCase;
++ptr;
while (ptr < end) {
if (!WTF::isASCIIAlphanumeric(*ptr))
break;
++ptr;
}
// Here's the shift
if (ptr < end) {
if ((!WTF::isASCII(*ptr)) || (*ptr == '\\') || (*ptr == '_') || (*ptr == '$'))
goto slowCase;
m_current = *ptr;
} else
m_current = 0;
m_code = ptr;
ASSERT(currentOffset() >= currentLineStartOffset());
// Create the identifier if needed
if (lexerFlags & LexexFlagsDontBuildKeywords)
tokenData->ident = 0;
else
tokenData->ident = makeLCharIdentifier(start, ptr - start);
tokenLocation->line = m_lineNumber;
tokenLocation->lineStartOffset = currentLineStartOffset();
tokenLocation->startOffset = offsetFromSourcePtr(start);
tokenLocation->endOffset = currentOffset();
ASSERT(tokenLocation->startOffset >= tokenLocation->lineStartOffset);
tokenRecord->m_startPosition = startPosition;
tokenRecord->m_endPosition = currentPosition();
m_lastToken = IDENT;
return IDENT;
slowCase:
return lex(tokenRecord, lexerFlags, strictMode);
}
} // namespace JSC
#endif // Lexer_h