1 |
|
2 |
|
3 | "use strict";
|
4 |
|
5 | var _classCallCheck = require("babel-runtime/helpers/class-call-check")["default"];
|
6 |
|
7 | var _interopRequireDefault = require("babel-runtime/helpers/interop-require-default")["default"];
|
8 |
|
9 | exports.__esModule = true;
|
10 |
|
11 | var _utilIdentifier = require("../util/identifier");
|
12 |
|
13 | var _types = require("./types");
|
14 |
|
15 | var _context = require("./context");
|
16 |
|
17 | var _utilLocation = require("../util/location");
|
18 |
|
19 | var _utilWhitespace = require("../util/whitespace");
|
20 |
|
21 | var _state = require("./state");
|
22 |
|
23 | var _state2 = _interopRequireDefault(_state);
|
24 |
|
25 |
|
26 |
|
27 |
|
28 |
|
29 |
|
30 | var Token = function Token(state) {
|
31 | _classCallCheck(this, Token);
|
32 |
|
33 | this.type = state.type;
|
34 | this.value = state.value;
|
35 | this.start = state.start;
|
36 | this.end = state.end;
|
37 | this.loc = new _utilLocation.SourceLocation(state.startLoc, state.endLoc);
|
38 | }
|
39 |
|
40 |
|
41 |
|
42 | ;
|
43 |
|
44 | exports.Token = Token;
|
45 | function codePointToString(code) {
|
46 |
|
47 | if (code <= 0xFFFF) {
|
48 | return String.fromCharCode(code);
|
49 | } else {
|
50 | return String.fromCharCode((code - 0x10000 >> 10) + 0xD800, (code - 0x10000 & 1023) + 0xDC00);
|
51 | }
|
52 | }
|
53 |
|
54 | var Tokenizer = (function () {
|
55 | function Tokenizer(options, input) {
|
56 | _classCallCheck(this, Tokenizer);
|
57 |
|
58 | this.state = new _state2["default"]();
|
59 | this.state.init(options, input);
|
60 | }
|
61 |
|
62 |
|
63 |
|
64 | Tokenizer.prototype.next = function next() {
|
65 | if (!this.isLookahead) {
|
66 | this.state.tokens.push(new Token(this.state));
|
67 | }
|
68 |
|
69 | this.state.lastTokEnd = this.state.end;
|
70 | this.state.lastTokStart = this.state.start;
|
71 | this.state.lastTokEndLoc = this.state.endLoc;
|
72 | this.state.lastTokStartLoc = this.state.startLoc;
|
73 | this.nextToken();
|
74 | };
|
75 |
|
76 |
|
77 |
|
78 | Tokenizer.prototype.eat = function eat(type) {
|
79 | if (this.match(type)) {
|
80 | this.next();
|
81 | return true;
|
82 | } else {
|
83 | return false;
|
84 | }
|
85 | };
|
86 |
|
87 |
|
88 |
|
89 | Tokenizer.prototype.match = function match(type) {
|
90 | return this.state.type === type;
|
91 | };
|
92 |
|
93 |
|
94 |
|
95 | Tokenizer.prototype.isKeyword = function isKeyword(word) {
|
96 | return _utilIdentifier.isKeyword(word);
|
97 | };
|
98 |
|
99 |
|
100 |
|
101 | Tokenizer.prototype.lookahead = function lookahead() {
|
102 | var old = this.state;
|
103 | this.state = old.clone(true);
|
104 |
|
105 | this.isLookahead = true;
|
106 | this.next();
|
107 | this.isLookahead = false;
|
108 |
|
109 | var curr = this.state.clone(true);
|
110 | this.state = old;
|
111 | return curr;
|
112 | };
|
113 |
|
114 |
|
115 |
|
116 |
|
117 | Tokenizer.prototype.setStrict = function setStrict(strict) {
|
118 | this.state.strict = strict;
|
119 | if (!this.match(_types.types.num) && !this.match(_types.types.string)) return;
|
120 | this.state.pos = this.state.start;
|
121 | while (this.state.pos < this.state.lineStart) {
|
122 | this.state.lineStart = this.input.lastIndexOf("\n", this.state.lineStart - 2) + 1;
|
123 | --this.state.curLine;
|
124 | }
|
125 | this.nextToken();
|
126 | };
|
127 |
|
128 | Tokenizer.prototype.curContext = function curContext() {
|
129 | return this.state.context[this.state.context.length - 1];
|
130 | };
|
131 |
|
132 |
|
133 |
|
134 |
|
135 | Tokenizer.prototype.nextToken = function nextToken() {
|
136 | var curContext = this.curContext();
|
137 | if (!curContext || !curContext.preserveSpace) this.skipSpace();
|
138 |
|
139 | this.state.containsOctal = false;
|
140 | this.state.octalPosition = null;
|
141 | this.state.start = this.state.pos;
|
142 | this.state.startLoc = this.state.curPosition();
|
143 | if (this.state.pos >= this.input.length) return this.finishToken(_types.types.eof);
|
144 |
|
145 | if (curContext.override) {
|
146 | return curContext.override(this);
|
147 | } else {
|
148 | return this.readToken(this.fullCharCodeAtPos());
|
149 | }
|
150 | };
|
151 |
|
152 | Tokenizer.prototype.readToken = function readToken(code) {
|
153 |
|
154 |
|
155 | if (_utilIdentifier.isIdentifierStart(code, true) || code === 92 ) {
|
156 | return this.readWord();
|
157 | } else {
|
158 | return this.getTokenFromCode(code);
|
159 | }
|
160 | };
|
161 |
|
162 | Tokenizer.prototype.fullCharCodeAtPos = function fullCharCodeAtPos() {
|
163 | var code = this.input.charCodeAt(this.state.pos);
|
164 | if (code <= 0xd7ff || code >= 0xe000) return code;
|
165 |
|
166 | var next = this.input.charCodeAt(this.state.pos + 1);
|
167 | return (code << 10) + next - 0x35fdc00;
|
168 | };
|
169 |
|
170 | Tokenizer.prototype.pushComment = function pushComment(block, text, start, end, startLoc, endLoc) {
|
171 | var comment = {
|
172 | type: block ? "CommentBlock" : "CommentLine",
|
173 | value: text,
|
174 | start: start,
|
175 | end: end,
|
176 | loc: new _utilLocation.SourceLocation(startLoc, endLoc)
|
177 | };
|
178 |
|
179 | if (!this.isLookahead) {
|
180 | this.state.tokens.push(comment);
|
181 | this.state.comments.push(comment);
|
182 | }
|
183 |
|
184 | this.addComment(comment);
|
185 | };
|
186 |
|
187 | Tokenizer.prototype.skipBlockComment = function skipBlockComment() {
|
188 | var startLoc = this.state.curPosition();
|
189 | var start = this.state.pos,
|
190 | end = this.input.indexOf("*/", this.state.pos += 2);
|
191 | if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment");
|
192 |
|
193 | this.state.pos = end + 2;
|
194 | _utilWhitespace.lineBreakG.lastIndex = start;
|
195 | var match = undefined;
|
196 | while ((match = _utilWhitespace.lineBreakG.exec(this.input)) && match.index < this.state.pos) {
|
197 | ++this.state.curLine;
|
198 | this.state.lineStart = match.index + match[0].length;
|
199 | }
|
200 |
|
201 | this.pushComment(true, this.input.slice(start + 2, end), start, this.state.pos, startLoc, this.state.curPosition());
|
202 | };
|
203 |
|
204 | Tokenizer.prototype.skipLineComment = function skipLineComment(startSkip) {
|
205 | var start = this.state.pos;
|
206 | var startLoc = this.state.curPosition();
|
207 | var ch = this.input.charCodeAt(this.state.pos += startSkip);
|
208 | while (this.state.pos < this.input.length && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) {
|
209 | ++this.state.pos;
|
210 | ch = this.input.charCodeAt(this.state.pos);
|
211 | }
|
212 |
|
213 | this.pushComment(false, this.input.slice(start + startSkip, this.state.pos), start, this.state.pos, startLoc, this.state.curPosition());
|
214 | };
|
215 |
|
216 |
|
217 |
|
218 |
|
219 | Tokenizer.prototype.skipSpace = function skipSpace() {
|
220 | loop: while (this.state.pos < this.input.length) {
|
221 | var ch = this.input.charCodeAt(this.state.pos);
|
222 | switch (ch) {
|
223 | case 32:case 160:
|
224 |
|
225 | ++this.state.pos;
|
226 | break;
|
227 |
|
228 | case 13:
|
229 | if (this.input.charCodeAt(this.state.pos + 1) === 10) {
|
230 | ++this.state.pos;
|
231 | }
|
232 |
|
233 | case 10:case 8232:case 8233:
|
234 | ++this.state.pos;
|
235 | ++this.state.curLine;
|
236 | this.state.lineStart = this.state.pos;
|
237 | break;
|
238 |
|
239 | case 47:
|
240 |
|
241 | switch (this.input.charCodeAt(this.state.pos + 1)) {
|
242 | case 42:
|
243 |
|
244 | this.skipBlockComment();
|
245 | break;
|
246 |
|
247 | case 47:
|
248 | this.skipLineComment(2);
|
249 | break;
|
250 |
|
251 | default:
|
252 | break loop;
|
253 | }
|
254 | break;
|
255 |
|
256 | default:
|
257 | if (ch > 8 && ch < 14 || ch >= 5760 && _utilWhitespace.nonASCIIwhitespace.test(String.fromCharCode(ch))) {
|
258 | ++this.state.pos;
|
259 | } else {
|
260 | break loop;
|
261 | }
|
262 | }
|
263 | }
|
264 | };
|
265 |
|
266 |
|
267 |
|
268 |
|
269 |
|
270 |
|
271 | Tokenizer.prototype.finishToken = function finishToken(type, val) {
|
272 | this.state.end = this.state.pos;
|
273 | this.state.endLoc = this.state.curPosition();
|
274 | var prevType = this.state.type;
|
275 | this.state.type = type;
|
276 | this.state.value = val;
|
277 |
|
278 | this.updateContext(prevType);
|
279 | };
|
280 |
|
281 |
|
282 |
|
283 |
|
284 |
|
285 |
|
286 |
|
287 |
|
288 |
|
289 |
|
290 |
|
291 | Tokenizer.prototype.readToken_dot = function readToken_dot() {
|
292 | var next = this.input.charCodeAt(this.state.pos + 1);
|
293 | if (next >= 48 && next <= 57) {
|
294 | return this.readNumber(true);
|
295 | }
|
296 |
|
297 | var next2 = this.input.charCodeAt(this.state.pos + 2);
|
298 | if (next === 46 && next2 === 46) {
|
299 |
|
300 | this.state.pos += 3;
|
301 | return this.finishToken(_types.types.ellipsis);
|
302 | } else {
|
303 | ++this.state.pos;
|
304 | return this.finishToken(_types.types.dot);
|
305 | }
|
306 | };
|
307 |
|
308 | Tokenizer.prototype.readToken_slash = function readToken_slash() {
|
309 |
|
310 | if (this.state.exprAllowed) {
|
311 | ++this.state.pos;
|
312 | return this.readRegexp();
|
313 | }
|
314 |
|
315 | var next = this.input.charCodeAt(this.state.pos + 1);
|
316 | if (next === 61) {
|
317 | return this.finishOp(_types.types.assign, 2);
|
318 | } else {
|
319 | return this.finishOp(_types.types.slash, 1);
|
320 | }
|
321 | };
|
322 |
|
323 | Tokenizer.prototype.readToken_mult_modulo = function readToken_mult_modulo(code) {
|
324 |
|
325 | var type = code === 42 ? _types.types.star : _types.types.modulo;
|
326 | var width = 1;
|
327 | var next = this.input.charCodeAt(this.state.pos + 1);
|
328 |
|
329 | if (next === 42 && this.hasPlugin("exponentiationOperator")) {
|
330 |
|
331 | width++;
|
332 | next = this.input.charCodeAt(this.state.pos + 2);
|
333 | type = _types.types.exponent;
|
334 | }
|
335 |
|
336 | if (next === 61) {
|
337 | width++;
|
338 | type = _types.types.assign;
|
339 | }
|
340 |
|
341 | return this.finishOp(type, width);
|
342 | };
|
343 |
|
344 | Tokenizer.prototype.readToken_pipe_amp = function readToken_pipe_amp(code) {
|
345 |
|
346 | var next = this.input.charCodeAt(this.state.pos + 1);
|
347 | if (next === code) return this.finishOp(code === 124 ? _types.types.logicalOR : _types.types.logicalAND, 2);
|
348 | if (next === 61) return this.finishOp(_types.types.assign, 2);
|
349 | return this.finishOp(code === 124 ? _types.types.bitwiseOR : _types.types.bitwiseAND, 1);
|
350 | };
|
351 |
|
352 | Tokenizer.prototype.readToken_caret = function readToken_caret() {
|
353 |
|
354 | var next = this.input.charCodeAt(this.state.pos + 1);
|
355 | if (next === 61) {
|
356 | return this.finishOp(_types.types.assign, 2);
|
357 | } else {
|
358 | return this.finishOp(_types.types.bitwiseXOR, 1);
|
359 | }
|
360 | };
|
361 |
|
362 | Tokenizer.prototype.readToken_plus_min = function readToken_plus_min(code) {
|
363 |
|
364 | var next = this.input.charCodeAt(this.state.pos + 1);
|
365 |
|
366 | if (next === code) {
|
367 | if (next === 45 && this.input.charCodeAt(this.state.pos + 2) === 62 && _utilWhitespace.lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.pos))) {
|
368 |
|
369 | this.skipLineComment(3);
|
370 | this.skipSpace();
|
371 | return this.nextToken();
|
372 | }
|
373 | return this.finishOp(_types.types.incDec, 2);
|
374 | }
|
375 |
|
376 | if (next === 61) {
|
377 | return this.finishOp(_types.types.assign, 2);
|
378 | } else {
|
379 | return this.finishOp(_types.types.plusMin, 1);
|
380 | }
|
381 | };
|
382 |
|
383 | Tokenizer.prototype.readToken_lt_gt = function readToken_lt_gt(code) {
|
384 |
|
385 | var next = this.input.charCodeAt(this.state.pos + 1);
|
386 | var size = 1;
|
387 |
|
388 | if (next === code) {
|
389 | size = code === 62 && this.input.charCodeAt(this.state.pos + 2) === 62 ? 3 : 2;
|
390 | if (this.input.charCodeAt(this.state.pos + size) === 61) return this.finishOp(_types.types.assign, size + 1);
|
391 | return this.finishOp(_types.types.bitShift, size);
|
392 | }
|
393 |
|
394 | if (next === 33 && code === 60 && this.input.charCodeAt(this.state.pos + 2) === 45 && this.input.charCodeAt(this.state.pos + 3) === 45) {
|
395 | if (this.inModule) this.unexpected();
|
396 |
|
397 | this.skipLineComment(4);
|
398 | this.skipSpace();
|
399 | return this.nextToken();
|
400 | }
|
401 |
|
402 | if (next === 61) {
|
403 | size = this.input.charCodeAt(this.state.pos + 2) === 61 ? 3 : 2;
|
404 | }
|
405 |
|
406 | return this.finishOp(_types.types.relational, size);
|
407 | };
|
408 |
|
409 | Tokenizer.prototype.readToken_eq_excl = function readToken_eq_excl(code) {
|
410 |
|
411 | var next = this.input.charCodeAt(this.state.pos + 1);
|
412 | if (next === 61) return this.finishOp(_types.types.equality, this.input.charCodeAt(this.state.pos + 2) === 61 ? 3 : 2);
|
413 | if (code === 61 && next === 62) {
|
414 |
|
415 | this.state.pos += 2;
|
416 | return this.finishToken(_types.types.arrow);
|
417 | }
|
418 | return this.finishOp(code === 61 ? _types.types.eq : _types.types.prefix, 1);
|
419 | };
|
420 |
|
421 | Tokenizer.prototype.getTokenFromCode = function getTokenFromCode(code) {
|
422 | switch (code) {
|
423 |
|
424 |
|
425 | case 46:
|
426 |
|
427 | return this.readToken_dot();
|
428 |
|
429 |
|
430 | case 40:
|
431 | ++this.state.pos;return this.finishToken(_types.types.parenL);
|
432 | case 41:
|
433 | ++this.state.pos;return this.finishToken(_types.types.parenR);
|
434 | case 59:
|
435 | ++this.state.pos;return this.finishToken(_types.types.semi);
|
436 | case 44:
|
437 | ++this.state.pos;return this.finishToken(_types.types.comma);
|
438 | case 91:
|
439 | ++this.state.pos;return this.finishToken(_types.types.bracketL);
|
440 | case 93:
|
441 | ++this.state.pos;return this.finishToken(_types.types.bracketR);
|
442 | case 123:
|
443 | ++this.state.pos;return this.finishToken(_types.types.braceL);
|
444 | case 125:
|
445 | ++this.state.pos;return this.finishToken(_types.types.braceR);
|
446 |
|
447 | case 58:
|
448 | if (this.hasPlugin("functionBind") && this.input.charCodeAt(this.state.pos + 1) === 58) {
|
449 | return this.finishOp(_types.types.doubleColon, 2);
|
450 | } else {
|
451 | ++this.state.pos;
|
452 | return this.finishToken(_types.types.colon);
|
453 | }
|
454 |
|
455 | case 63:
|
456 | ++this.state.pos;return this.finishToken(_types.types.question);
|
457 | case 64:
|
458 | ++this.state.pos;return this.finishToken(_types.types.at);
|
459 |
|
460 | case 96:
|
461 |
|
462 | ++this.state.pos;
|
463 | return this.finishToken(_types.types.backQuote);
|
464 |
|
465 | case 48:
|
466 |
|
467 | var next = this.input.charCodeAt(this.state.pos + 1);
|
468 | if (next === 120 || next === 88) return this.readRadixNumber(16);
|
469 | if (next === 111 || next === 79) return this.readRadixNumber(8);
|
470 | if (next === 98 || next === 66) return this.readRadixNumber(2);
|
471 |
|
472 |
|
473 | case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57:
|
474 |
|
475 | return this.readNumber(false);
|
476 |
|
477 |
|
478 | case 34:case 39:
|
479 |
|
480 | return this.readString(code);
|
481 |
|
482 |
|
483 |
|
484 |
|
485 |
|
486 |
|
487 | case 47:
|
488 |
|
489 | return this.readToken_slash();
|
490 |
|
491 | case 37:case 42:
|
492 |
|
493 | return this.readToken_mult_modulo(code);
|
494 |
|
495 | case 124:case 38:
|
496 |
|
497 | return this.readToken_pipe_amp(code);
|
498 |
|
499 | case 94:
|
500 |
|
501 | return this.readToken_caret();
|
502 |
|
503 | case 43:case 45:
|
504 |
|
505 | return this.readToken_plus_min(code);
|
506 |
|
507 | case 60:case 62:
|
508 |
|
509 | return this.readToken_lt_gt(code);
|
510 |
|
511 | case 61:case 33:
|
512 |
|
513 | return this.readToken_eq_excl(code);
|
514 |
|
515 | case 126:
|
516 |
|
517 | return this.finishOp(_types.types.prefix, 1);
|
518 | }
|
519 |
|
520 | this.raise(this.state.pos, "Unexpected character '" + codePointToString(code) + "'");
|
521 | };
|
522 |
|
523 | Tokenizer.prototype.finishOp = function finishOp(type, size) {
|
524 | var str = this.input.slice(this.state.pos, this.state.pos + size);
|
525 | this.state.pos += size;
|
526 | return this.finishToken(type, str);
|
527 | };
|
528 |
|
529 | Tokenizer.prototype.readRegexp = function readRegexp() {
|
530 | var escaped = undefined,
|
531 | inClass = undefined,
|
532 | start = this.state.pos;
|
533 | for (;;) {
|
534 | if (this.state.pos >= this.input.length) this.raise(start, "Unterminated regular expression");
|
535 | var ch = this.input.charAt(this.state.pos);
|
536 | if (_utilWhitespace.lineBreak.test(ch)) {
|
537 | this.raise(start, "Unterminated regular expression");
|
538 | }
|
539 | if (escaped) {
|
540 | escaped = false;
|
541 | } else {
|
542 | if (ch === "[") {
|
543 | inClass = true;
|
544 | } else if (ch === "]" && inClass) {
|
545 | inClass = false;
|
546 | } else if (ch === "/" && !inClass) {
|
547 | break;
|
548 | }
|
549 | escaped = ch === "\\";
|
550 | }
|
551 | ++this.state.pos;
|
552 | }
|
553 | var content = this.input.slice(start, this.state.pos);
|
554 | ++this.state.pos;
|
555 |
|
556 |
|
557 | var mods = this.readWord1();
|
558 | if (mods) {
|
559 | var validFlags = /^[gmsiyu]*$/;
|
560 | if (!validFlags.test(mods)) this.raise(start, "Invalid regular expression flag");
|
561 | }
|
562 | return this.finishToken(_types.types.regexp, {
|
563 | pattern: content,
|
564 | flags: mods
|
565 | });
|
566 | };
|
567 |
|
568 |
|
569 |
|
570 |
|
571 |
|
572 | Tokenizer.prototype.readInt = function readInt(radix, len) {
|
573 | var start = this.state.pos,
|
574 | total = 0;
|
575 | for (var i = 0, e = len == null ? Infinity : len; i < e; ++i) {
|
576 | var code = this.input.charCodeAt(this.state.pos),
|
577 | val = undefined;
|
578 | if (code >= 97) {
|
579 | val = code - 97 + 10;
|
580 | } else if (code >= 65) {
|
581 | val = code - 65 + 10;
|
582 | } else if (code >= 48 && code <= 57) {
|
583 | val = code - 48;
|
584 | } else {
|
585 | val = Infinity;
|
586 | }
|
587 | if (val >= radix) break;
|
588 | ++this.state.pos;
|
589 | total = total * radix + val;
|
590 | }
|
591 | if (this.state.pos === start || len != null && this.state.pos - start !== len) return null;
|
592 |
|
593 | return total;
|
594 | };
|
595 |
|
596 | Tokenizer.prototype.readRadixNumber = function readRadixNumber(radix) {
|
597 | this.state.pos += 2;
|
598 | var val = this.readInt(radix);
|
599 | if (val == null) this.raise(this.state.start + 2, "Expected number in radix " + radix);
|
600 | if (_utilIdentifier.isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number");
|
601 | return this.finishToken(_types.types.num, val);
|
602 | };
|
603 |
|
604 |
|
605 |
|
606 | Tokenizer.prototype.readNumber = function readNumber(startsWithDot) {
|
607 | var start = this.state.pos,
|
608 | isFloat = false,
|
609 | octal = this.input.charCodeAt(this.state.pos) === 48;
|
610 | if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number");
|
611 | var next = this.input.charCodeAt(this.state.pos);
|
612 | if (next === 46) {
|
613 |
|
614 | ++this.state.pos;
|
615 | this.readInt(10);
|
616 | isFloat = true;
|
617 | next = this.input.charCodeAt(this.state.pos);
|
618 | }
|
619 | if (next === 69 || next === 101) {
|
620 |
|
621 | next = this.input.charCodeAt(++this.state.pos);
|
622 | if (next === 43 || next === 45) ++this.state.pos;
|
623 | if (this.readInt(10) === null) this.raise(start, "Invalid number");
|
624 | isFloat = true;
|
625 | }
|
626 | if (_utilIdentifier.isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number");
|
627 |
|
628 | var str = this.input.slice(start, this.state.pos),
|
629 | val = undefined;
|
630 | if (isFloat) {
|
631 | val = parseFloat(str);
|
632 | } else if (!octal || str.length === 1) {
|
633 | val = parseInt(str, 10);
|
634 | } else if (/[89]/.test(str) || this.state.strict) {
|
635 | this.raise(start, "Invalid number");
|
636 | } else {
|
637 | val = parseInt(str, 8);
|
638 | }
|
639 | return this.finishToken(_types.types.num, val);
|
640 | };
|
641 |
|
642 |
|
643 |
|
644 | Tokenizer.prototype.readCodePoint = function readCodePoint() {
|
645 | var ch = this.input.charCodeAt(this.state.pos),
|
646 | code = undefined;
|
647 |
|
648 | if (ch === 123) {
|
649 | var codePos = ++this.state.pos;
|
650 | code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos);
|
651 | ++this.state.pos;
|
652 | if (code > 0x10FFFF) this.raise(codePos, "Code point out of bounds");
|
653 | } else {
|
654 | code = this.readHexChar(4);
|
655 | }
|
656 | return code;
|
657 | };
|
658 |
|
659 | Tokenizer.prototype.readString = function readString(quote) {
|
660 | var out = "",
|
661 | chunkStart = ++this.state.pos;
|
662 | for (;;) {
|
663 | if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated string constant");
|
664 | var ch = this.input.charCodeAt(this.state.pos);
|
665 | if (ch === quote) break;
|
666 | if (ch === 92) {
|
667 |
|
668 | out += this.input.slice(chunkStart, this.state.pos);
|
669 | out += this.readEscapedChar(false);
|
670 | chunkStart = this.state.pos;
|
671 | } else {
|
672 | if (_utilWhitespace.isNewLine(ch)) this.raise(this.state.start, "Unterminated string constant");
|
673 | ++this.state.pos;
|
674 | }
|
675 | }
|
676 | out += this.input.slice(chunkStart, this.state.pos++);
|
677 | return this.finishToken(_types.types.string, out);
|
678 | };
|
679 |
|
680 |
|
681 |
|
682 | Tokenizer.prototype.readTmplToken = function readTmplToken() {
|
683 | var out = "",
|
684 | chunkStart = this.state.pos;
|
685 | for (;;) {
|
686 | if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated template");
|
687 | var ch = this.input.charCodeAt(this.state.pos);
|
688 | if (ch === 96 || ch === 36 && this.input.charCodeAt(this.state.pos + 1) === 123) {
|
689 |
|
690 | if (this.state.pos === this.state.start && this.match(_types.types.template)) {
|
691 | if (ch === 36) {
|
692 | this.state.pos += 2;
|
693 | return this.finishToken(_types.types.dollarBraceL);
|
694 | } else {
|
695 | ++this.state.pos;
|
696 | return this.finishToken(_types.types.backQuote);
|
697 | }
|
698 | }
|
699 | out += this.input.slice(chunkStart, this.state.pos);
|
700 | return this.finishToken(_types.types.template, out);
|
701 | }
|
702 | if (ch === 92) {
|
703 |
|
704 | out += this.input.slice(chunkStart, this.state.pos);
|
705 | out += this.readEscapedChar(true);
|
706 | chunkStart = this.state.pos;
|
707 | } else if (_utilWhitespace.isNewLine(ch)) {
|
708 | out += this.input.slice(chunkStart, this.state.pos);
|
709 | ++this.state.pos;
|
710 | switch (ch) {
|
711 | case 13:
|
712 | if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos;
|
713 | case 10:
|
714 | out += "\n";
|
715 | break;
|
716 | default:
|
717 | out += String.fromCharCode(ch);
|
718 | break;
|
719 | }
|
720 | ++this.state.curLine;
|
721 | this.state.lineStart = this.state.pos;
|
722 | chunkStart = this.state.pos;
|
723 | } else {
|
724 | ++this.state.pos;
|
725 | }
|
726 | }
|
727 | };
|
728 |
|
729 |
|
730 |
|
731 | Tokenizer.prototype.readEscapedChar = function readEscapedChar(inTemplate) {
|
732 | var ch = this.input.charCodeAt(++this.state.pos);
|
733 | ++this.state.pos;
|
734 | switch (ch) {
|
735 | case 110:
|
736 | return "\n";
|
737 | case 114:
|
738 | return "\r";
|
739 | case 120:
|
740 | return String.fromCharCode(this.readHexChar(2));
|
741 | case 117:
|
742 | return codePointToString(this.readCodePoint());
|
743 | case 116:
|
744 | return "\t";
|
745 | case 98:
|
746 | return "\b";
|
747 | case 118:
|
748 | return "\u000b";
|
749 | case 102:
|
750 | return "\f";
|
751 | case 13:
|
752 | if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos;
|
753 | case 10:
|
754 |
|
755 | this.state.lineStart = this.state.pos;
|
756 | ++this.state.curLine;
|
757 | return "";
|
758 | default:
|
759 | if (ch >= 48 && ch <= 55) {
|
760 | var octalStr = this.input.substr(this.state.pos - 1, 3).match(/^[0-7]+/)[0];
|
761 | var octal = parseInt(octalStr, 8);
|
762 | if (octal > 255) {
|
763 | octalStr = octalStr.slice(0, -1);
|
764 | octal = parseInt(octalStr, 8);
|
765 | }
|
766 | if (octal > 0) {
|
767 | if (!this.state.containsOctal) {
|
768 | this.state.containsOctal = true;
|
769 | this.state.octalPosition = this.state.pos - 2;
|
770 | }
|
771 | if (this.state.strict || inTemplate) {
|
772 | this.raise(this.state.pos - 2, "Octal literal in strict mode");
|
773 | }
|
774 | }
|
775 | this.state.pos += octalStr.length - 1;
|
776 | return String.fromCharCode(octal);
|
777 | }
|
778 | return String.fromCharCode(ch);
|
779 | }
|
780 | };
|
781 |
|
782 |
|
783 |
|
784 | Tokenizer.prototype.readHexChar = function readHexChar(len) {
|
785 | var codePos = this.state.pos;
|
786 | var n = this.readInt(16, len);
|
787 | if (n === null) this.raise(codePos, "Bad character escape sequence");
|
788 | return n;
|
789 | };
|
790 |
|
791 |
|
792 |
|
793 |
|
794 |
|
795 |
|
796 |
|
797 | Tokenizer.prototype.readWord1 = function readWord1() {
|
798 | this.state.containsEsc = false;
|
799 | var word = "",
|
800 | first = true,
|
801 | chunkStart = this.state.pos;
|
802 | while (this.state.pos < this.input.length) {
|
803 | var ch = this.fullCharCodeAtPos();
|
804 | if (_utilIdentifier.isIdentifierChar(ch, true)) {
|
805 | this.state.pos += ch <= 0xffff ? 1 : 2;
|
806 | } else if (ch === 92) {
|
807 |
|
808 | this.state.containsEsc = true;
|
809 |
|
810 | word += this.input.slice(chunkStart, this.state.pos);
|
811 | var escStart = this.state.pos;
|
812 |
|
813 | if (this.input.charCodeAt(++this.state.pos) !== 117) {
|
814 |
|
815 | this.raise(this.state.pos, "Expecting Unicode escape sequence \\uXXXX");
|
816 | }
|
817 |
|
818 | ++this.state.pos;
|
819 | var esc = this.readCodePoint();
|
820 | if (!(first ? _utilIdentifier.isIdentifierStart : _utilIdentifier.isIdentifierChar)(esc, true)) {
|
821 | this.raise(escStart, "Invalid Unicode escape");
|
822 | }
|
823 |
|
824 | word += codePointToString(esc);
|
825 | chunkStart = this.state.pos;
|
826 | } else {
|
827 | break;
|
828 | }
|
829 | first = false;
|
830 | }
|
831 | return word + this.input.slice(chunkStart, this.state.pos);
|
832 | };
|
833 |
|
834 |
|
835 |
|
836 |
|
837 | Tokenizer.prototype.readWord = function readWord() {
|
838 | var word = this.readWord1();
|
839 | var type = _types.types.name;
|
840 | if (!this.state.containsEsc && this.isKeyword(word)) {
|
841 | type = _types.keywords[word];
|
842 | }
|
843 | return this.finishToken(type, word);
|
844 | };
|
845 |
|
846 | Tokenizer.prototype.braceIsBlock = function braceIsBlock(prevType) {
|
847 | if (prevType === _types.types.colon) {
|
848 | var _parent = this.curContext();
|
849 | if (_parent === _context.types.b_stat || _parent === _context.types.b_expr) {
|
850 | return !_parent.isExpr;
|
851 | }
|
852 | }
|
853 |
|
854 | if (prevType === _types.types._return) {
|
855 | return _utilWhitespace.lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start));
|
856 | }
|
857 |
|
858 | if (prevType === _types.types._else || prevType === _types.types.semi || prevType === _types.types.eof || prevType === _types.types.parenR) {
|
859 | return true;
|
860 | }
|
861 |
|
862 | if (prevType === _types.types.braceL) {
|
863 | return this.curContext() === _context.types.b_stat;
|
864 | }
|
865 |
|
866 | return !this.state.exprAllowed;
|
867 | };
|
868 |
|
869 | Tokenizer.prototype.updateContext = function updateContext(prevType) {
|
870 | var update = undefined,
|
871 | type = this.state.type;
|
872 | if (type.keyword && prevType === _types.types.dot) {
|
873 | this.state.exprAllowed = false;
|
874 | } else if (update = type.updateContext) {
|
875 | update.call(this, prevType);
|
876 | } else {
|
877 | this.state.exprAllowed = type.beforeExpr;
|
878 | }
|
879 | };
|
880 |
|
881 | return Tokenizer;
|
882 | })();
|
883 |
|
884 | exports["default"] = Tokenizer; |
\ | No newline at end of file |