UNPKG

7.75 kBJavaScriptView Raw
1"use strict";
2Object.defineProperty(exports, "__esModule", { value: true });
3exports.createTokenizer = void 0;
4/**
5 * Creates a tokenizer for the specified source.
6 *
7 * @param source
8 */
9function createTokenizer(source) {
10 const end = source.length;
11 let pos = 0;
12 let type = 'EOF';
13 let value = '';
14 let flags = 0 /* None */;
15 // These are used to greedily skip as much as possible.
16 // Whenever we reach a paren, we increment these.
17 let parenLeft = 0;
18 let parenRight = 0;
19 return {
20 next,
21 done,
22 };
23 /**
24 * Advances the tokenizer and returns the next token.
25 */
26 function next(nextFlags = 0 /* None */) {
27 flags = nextFlags;
28 advance();
29 return createToken();
30 }
31 /**
32 * Advances the tokenizer state.
33 */
34 function advance() {
35 value = '';
36 type = 'EOF';
37 while (true) {
38 if (pos >= end) {
39 return (type = 'EOF');
40 }
41 let ch = source.charAt(pos);
42 // Whitespace is irrelevant
43 if (isWhiteSpace(ch)) {
44 pos++;
45 continue;
46 }
47 switch (ch) {
48 case '(':
49 pos++;
50 parenLeft++;
51 return (type = ch);
52 case ')':
53 pos++;
54 parenRight++;
55 return (type = ch);
56 case '*':
57 pos++;
58 return (type = ch);
59 case ',':
60 pos++;
61 return (type = ch);
62 case '=':
63 pos++;
64 if ((flags & 1 /* Dumb */) === 0) {
65 // Not in dumb-mode, so attempt to skip.
66 skipExpression();
67 }
68 // We need to know that there's a default value so we can
69 // skip it if it does not exist when resolving.
70 return (type = ch);
71 case '/':
72 pos++;
73 const nextCh = source.charAt(pos);
74 if (nextCh === '/') {
75 skipUntil((c) => c === '\n', true);
76 pos++;
77 }
78 if (nextCh === '*') {
79 skipUntil((c) => {
80 const closing = source.charAt(pos + 1);
81 return c === '*' && closing === '/';
82 }, true);
83 pos++;
84 }
85 continue;
86 default:
87 // Scans an identifier.
88 if (isIdentifierStart(ch)) {
89 scanIdentifier();
90 return type;
91 }
92 // Elegantly skip over tokens we don't care about.
93 pos++;
94 }
95 }
96 }
97 /**
98 * Scans an identifier, given it's already been proven
99 * we are ready to do so.
100 */
101 function scanIdentifier() {
102 const identStart = source.charAt(pos);
103 const start = ++pos;
104 while (isIdentifierPart(source.charAt(pos))) {
105 pos++;
106 }
107 value = '' + identStart + source.substring(start, pos);
108 type = value === 'function' || value === 'class' ? value : 'ident';
109 if (type !== 'ident') {
110 value = '';
111 }
112 return value;
113 }
114 /**
115 * Skips everything until the next comma or the end of the parameter list.
116 * Checks the parenthesis balance so we correctly skip function calls.
117 */
118 function skipExpression() {
119 skipUntil((ch) => {
120 const isAtRoot = parenLeft === parenRight + 1;
121 if (ch === ',' && isAtRoot) {
122 return true;
123 }
124 if (ch === '(') {
125 parenLeft++;
126 return false;
127 }
128 if (ch === ')') {
129 parenRight++;
130 if (isAtRoot) {
131 return true;
132 }
133 }
134 return false;
135 });
136 }
137 /**
138 * Skips strings and whilespace until the predicate is true.
139 *
140 * @param callback stops skipping when this returns `true`.
141 * @param dumb if `true`, does not skip whitespace and strings;
142 * it only stops once the callback returns `true`.
143 */
144 function skipUntil(callback, dumb = false) {
145 while (pos < source.length) {
146 let ch = source.charAt(pos);
147 if (callback(ch)) {
148 return;
149 }
150 if (!dumb) {
151 if (isWhiteSpace(ch)) {
152 pos++;
153 continue;
154 }
155 if (isStringQuote(ch)) {
156 skipString();
157 continue;
158 }
159 }
160 pos++;
161 }
162 }
163 /**
164 * Given the current position is at a string quote, skips the entire string.
165 */
166 function skipString() {
167 const quote = source.charAt(pos);
168 pos++;
169 while (pos < source.length) {
170 const ch = source.charAt(pos);
171 const prev = source.charAt(pos - 1);
172 // Checks if the quote was escaped.
173 if (ch === quote && prev !== '\\') {
174 pos++;
175 return;
176 }
177 // Template strings are a bit tougher, we want to skip the interpolated values.
178 if (quote === '`') {
179 const next = source.charAt(pos + 1);
180 if (next === '$') {
181 const afterDollar = source.charAt(pos + 2);
182 if (afterDollar === '{') {
183 // This is the start of an interpolation; skip the ${
184 pos = pos + 2;
185 // Skip strings and whitespace until we reach the ending }.
186 // This includes skipping nested interpolated strings. :D
187 skipUntil((ch) => ch === '}');
188 }
189 }
190 }
191 pos++;
192 }
193 }
194 /**
195 * Creates a token from the current state.
196 */
197 function createToken() {
198 if (value) {
199 return { value, type };
200 }
201 return { type };
202 }
203 /**
204 * Determines if we are done parsing.
205 */
206 function done() {
207 return type === 'EOF';
208 }
209}
210exports.createTokenizer = createTokenizer;
211/**
212 * Determines if the given character is a whitespace character.
213 *
214 * @param {string} ch
215 * @return {boolean}
216 */
217function isWhiteSpace(ch) {
218 switch (ch) {
219 case '\r':
220 case '\n':
221 case ' ':
222 return true;
223 }
224 return false;
225}
226/**
227 * Determines if the specified character is a string quote.
228 * @param {string} ch
229 * @return {boolean}
230 */
231function isStringQuote(ch) {
232 switch (ch) {
233 case "'":
234 case '"':
235 case '`':
236 return true;
237 }
238 return false;
239}
240// NOTE: I've added the `.` character so that member expression paths
241// are seen as identifiers. This is so we don't get a constructor token for
242// stuff like `MyClass.prototype.constructor()`
243const IDENT_START_EXPR = /^[_$a-zA-Z\xA0-\uFFFF]$/;
244const IDENT_PART_EXPR = /^[._$a-zA-Z0-9\xA0-\uFFFF]$/;
245/**
246 * Determines if the character is a valid JS identifier start character.
247 */
248function isIdentifierStart(ch) {
249 return IDENT_START_EXPR.test(ch);
250}
251/**
252 * Determines if the character is a valid JS identifier start character.
253 */
254function isIdentifierPart(ch) {
255 return IDENT_PART_EXPR.test(ch);
256}
257//# sourceMappingURL=function-tokenizer.js.map
\No newline at end of file