1 | ;
|
2 | Object.defineProperty(exports, "__esModule", { value: true });
|
3 | exports.createTokenizer = void 0;
|
4 | /**
|
5 | * Creates a tokenizer for the specified source.
|
6 | *
|
7 | * @param source
|
8 | */
|
9 | function createTokenizer(source) {
|
10 | const end = source.length;
|
11 | let pos = 0;
|
12 | let type = 'EOF';
|
13 | let value = '';
|
14 | let flags = 0 /* None */;
|
15 | // These are used to greedily skip as much as possible.
|
16 | // Whenever we reach a paren, we increment these.
|
17 | let parenLeft = 0;
|
18 | let parenRight = 0;
|
19 | return {
|
20 | next,
|
21 | done,
|
22 | };
|
23 | /**
|
24 | * Advances the tokenizer and returns the next token.
|
25 | */
|
26 | function next(nextFlags = 0 /* None */) {
|
27 | flags = nextFlags;
|
28 | advance();
|
29 | return createToken();
|
30 | }
|
31 | /**
|
32 | * Advances the tokenizer state.
|
33 | */
|
34 | function advance() {
|
35 | value = '';
|
36 | type = 'EOF';
|
37 | while (true) {
|
38 | if (pos >= end) {
|
39 | return (type = 'EOF');
|
40 | }
|
41 | let ch = source.charAt(pos);
|
42 | // Whitespace is irrelevant
|
43 | if (isWhiteSpace(ch)) {
|
44 | pos++;
|
45 | continue;
|
46 | }
|
47 | switch (ch) {
|
48 | case '(':
|
49 | pos++;
|
50 | parenLeft++;
|
51 | return (type = ch);
|
52 | case ')':
|
53 | pos++;
|
54 | parenRight++;
|
55 | return (type = ch);
|
56 | case '*':
|
57 | pos++;
|
58 | return (type = ch);
|
59 | case ',':
|
60 | pos++;
|
61 | return (type = ch);
|
62 | case '=':
|
63 | pos++;
|
64 | if ((flags & 1 /* Dumb */) === 0) {
|
65 | // Not in dumb-mode, so attempt to skip.
|
66 | skipExpression();
|
67 | }
|
68 | // We need to know that there's a default value so we can
|
69 | // skip it if it does not exist when resolving.
|
70 | return (type = ch);
|
71 | case '/':
|
72 | pos++;
|
73 | const nextCh = source.charAt(pos);
|
74 | if (nextCh === '/') {
|
75 | skipUntil((c) => c === '\n', true);
|
76 | pos++;
|
77 | }
|
78 | if (nextCh === '*') {
|
79 | skipUntil((c) => {
|
80 | const closing = source.charAt(pos + 1);
|
81 | return c === '*' && closing === '/';
|
82 | }, true);
|
83 | pos++;
|
84 | }
|
85 | continue;
|
86 | default:
|
87 | // Scans an identifier.
|
88 | if (isIdentifierStart(ch)) {
|
89 | scanIdentifier();
|
90 | return type;
|
91 | }
|
92 | // Elegantly skip over tokens we don't care about.
|
93 | pos++;
|
94 | }
|
95 | }
|
96 | }
|
97 | /**
|
98 | * Scans an identifier, given it's already been proven
|
99 | * we are ready to do so.
|
100 | */
|
101 | function scanIdentifier() {
|
102 | const identStart = source.charAt(pos);
|
103 | const start = ++pos;
|
104 | while (isIdentifierPart(source.charAt(pos))) {
|
105 | pos++;
|
106 | }
|
107 | value = '' + identStart + source.substring(start, pos);
|
108 | type = value === 'function' || value === 'class' ? value : 'ident';
|
109 | if (type !== 'ident') {
|
110 | value = '';
|
111 | }
|
112 | return value;
|
113 | }
|
114 | /**
|
115 | * Skips everything until the next comma or the end of the parameter list.
|
116 | * Checks the parenthesis balance so we correctly skip function calls.
|
117 | */
|
118 | function skipExpression() {
|
119 | skipUntil((ch) => {
|
120 | const isAtRoot = parenLeft === parenRight + 1;
|
121 | if (ch === ',' && isAtRoot) {
|
122 | return true;
|
123 | }
|
124 | if (ch === '(') {
|
125 | parenLeft++;
|
126 | return false;
|
127 | }
|
128 | if (ch === ')') {
|
129 | parenRight++;
|
130 | if (isAtRoot) {
|
131 | return true;
|
132 | }
|
133 | }
|
134 | return false;
|
135 | });
|
136 | }
|
137 | /**
|
138 | * Skips strings and whilespace until the predicate is true.
|
139 | *
|
140 | * @param callback stops skipping when this returns `true`.
|
141 | * @param dumb if `true`, does not skip whitespace and strings;
|
142 | * it only stops once the callback returns `true`.
|
143 | */
|
144 | function skipUntil(callback, dumb = false) {
|
145 | while (pos < source.length) {
|
146 | let ch = source.charAt(pos);
|
147 | if (callback(ch)) {
|
148 | return;
|
149 | }
|
150 | if (!dumb) {
|
151 | if (isWhiteSpace(ch)) {
|
152 | pos++;
|
153 | continue;
|
154 | }
|
155 | if (isStringQuote(ch)) {
|
156 | skipString();
|
157 | continue;
|
158 | }
|
159 | }
|
160 | pos++;
|
161 | }
|
162 | }
|
163 | /**
|
164 | * Given the current position is at a string quote, skips the entire string.
|
165 | */
|
166 | function skipString() {
|
167 | const quote = source.charAt(pos);
|
168 | pos++;
|
169 | while (pos < source.length) {
|
170 | const ch = source.charAt(pos);
|
171 | const prev = source.charAt(pos - 1);
|
172 | // Checks if the quote was escaped.
|
173 | if (ch === quote && prev !== '\\') {
|
174 | pos++;
|
175 | return;
|
176 | }
|
177 | // Template strings are a bit tougher, we want to skip the interpolated values.
|
178 | if (quote === '`') {
|
179 | const next = source.charAt(pos + 1);
|
180 | if (next === '$') {
|
181 | const afterDollar = source.charAt(pos + 2);
|
182 | if (afterDollar === '{') {
|
183 | // This is the start of an interpolation; skip the ${
|
184 | pos = pos + 2;
|
185 | // Skip strings and whitespace until we reach the ending }.
|
186 | // This includes skipping nested interpolated strings. :D
|
187 | skipUntil((ch) => ch === '}');
|
188 | }
|
189 | }
|
190 | }
|
191 | pos++;
|
192 | }
|
193 | }
|
194 | /**
|
195 | * Creates a token from the current state.
|
196 | */
|
197 | function createToken() {
|
198 | if (value) {
|
199 | return { value, type };
|
200 | }
|
201 | return { type };
|
202 | }
|
203 | /**
|
204 | * Determines if we are done parsing.
|
205 | */
|
206 | function done() {
|
207 | return type === 'EOF';
|
208 | }
|
209 | }
|
210 | exports.createTokenizer = createTokenizer;
|
211 | /**
|
212 | * Determines if the given character is a whitespace character.
|
213 | *
|
214 | * @param {string} ch
|
215 | * @return {boolean}
|
216 | */
|
217 | function isWhiteSpace(ch) {
|
218 | switch (ch) {
|
219 | case '\r':
|
220 | case '\n':
|
221 | case ' ':
|
222 | return true;
|
223 | }
|
224 | return false;
|
225 | }
|
226 | /**
|
227 | * Determines if the specified character is a string quote.
|
228 | * @param {string} ch
|
229 | * @return {boolean}
|
230 | */
|
231 | function isStringQuote(ch) {
|
232 | switch (ch) {
|
233 | case "'":
|
234 | case '"':
|
235 | case '`':
|
236 | return true;
|
237 | }
|
238 | return false;
|
239 | }
|
240 | // NOTE: I've added the `.` character so that member expression paths
|
241 | // are seen as identifiers. This is so we don't get a constructor token for
|
242 | // stuff like `MyClass.prototype.constructor()`
|
243 | const IDENT_START_EXPR = /^[_$a-zA-Z\xA0-\uFFFF]$/;
|
244 | const IDENT_PART_EXPR = /^[._$a-zA-Z0-9\xA0-\uFFFF]$/;
|
245 | /**
|
246 | * Determines if the character is a valid JS identifier start character.
|
247 | */
|
248 | function isIdentifierStart(ch) {
|
249 | return IDENT_START_EXPR.test(ch);
|
250 | }
|
251 | /**
|
252 | * Determines if the character is a valid JS identifier start character.
|
253 | */
|
254 | function isIdentifierPart(ch) {
|
255 | return IDENT_PART_EXPR.test(ch);
|
256 | }
|
257 | //# sourceMappingURL=function-tokenizer.js.map |
\ | No newline at end of file |