UNPKG

8.23 kBJavaScriptView Raw
1
2
3
4import { TokenType as tt} from "./parser/tokenizer/types";
5import isAsyncOperation from "./util/isAsyncOperation";
6
7
8
9
10
11
12export default class TokenProcessor {
13 __init() {this.resultCode = ""}
14 __init2() {this.tokenIndex = 0}
15
16 constructor(
17 code,
18 tokens,
19 isFlowEnabled,
20 helperManager,
21 ) {;this.code = code;this.tokens = tokens;this.isFlowEnabled = isFlowEnabled;this.helperManager = helperManager;TokenProcessor.prototype.__init.call(this);TokenProcessor.prototype.__init2.call(this);}
22
23 /**
24 * Make a new TokenProcessor for things like lookahead.
25 */
26 snapshot() {
27 return {resultCode: this.resultCode, tokenIndex: this.tokenIndex};
28 }
29
30 restoreToSnapshot(snapshot) {
31 this.resultCode = snapshot.resultCode;
32 this.tokenIndex = snapshot.tokenIndex;
33 }
34
35 getResultCodeIndex() {
36 return this.resultCode.length;
37 }
38
39 reset() {
40 this.resultCode = "";
41 this.tokenIndex = 0;
42 }
43
44 matchesContextualAtIndex(index, contextualKeyword) {
45 return (
46 this.matches1AtIndex(index, tt.name) &&
47 this.tokens[index].contextualKeyword === contextualKeyword
48 );
49 }
50
51 identifierNameAtIndex(index) {
52 // TODO: We need to process escapes since technically you can have unicode escapes in variable
53 // names.
54 return this.identifierNameForToken(this.tokens[index]);
55 }
56
57 identifierName() {
58 return this.identifierNameForToken(this.currentToken());
59 }
60
61 identifierNameForToken(token) {
62 return this.code.slice(token.start, token.end);
63 }
64
65 rawCodeForToken(token) {
66 return this.code.slice(token.start, token.end);
67 }
68
69 stringValueAtIndex(index) {
70 return this.stringValueForToken(this.tokens[index]);
71 }
72
73 stringValue() {
74 return this.stringValueForToken(this.currentToken());
75 }
76
77 stringValueForToken(token) {
78 // This is used to identify when two imports are the same and to resolve TypeScript enum keys.
79 // Ideally we'd process escapes within the strings, but for now we pretty much take the raw
80 // code.
81 return this.code.slice(token.start + 1, token.end - 1);
82 }
83
84 matches1AtIndex(index, t1) {
85 return this.tokens[index].type === t1;
86 }
87
88 matches2AtIndex(index, t1, t2) {
89 return this.tokens[index].type === t1 && this.tokens[index + 1].type === t2;
90 }
91
92 matches3AtIndex(index, t1, t2, t3) {
93 return (
94 this.tokens[index].type === t1 &&
95 this.tokens[index + 1].type === t2 &&
96 this.tokens[index + 2].type === t3
97 );
98 }
99
100 matches1(t1) {
101 return this.tokens[this.tokenIndex].type === t1;
102 }
103
104 matches2(t1, t2) {
105 return this.tokens[this.tokenIndex].type === t1 && this.tokens[this.tokenIndex + 1].type === t2;
106 }
107
108 matches3(t1, t2, t3) {
109 return (
110 this.tokens[this.tokenIndex].type === t1 &&
111 this.tokens[this.tokenIndex + 1].type === t2 &&
112 this.tokens[this.tokenIndex + 2].type === t3
113 );
114 }
115
116 matches4(t1, t2, t3, t4) {
117 return (
118 this.tokens[this.tokenIndex].type === t1 &&
119 this.tokens[this.tokenIndex + 1].type === t2 &&
120 this.tokens[this.tokenIndex + 2].type === t3 &&
121 this.tokens[this.tokenIndex + 3].type === t4
122 );
123 }
124
125 matches5(t1, t2, t3, t4, t5) {
126 return (
127 this.tokens[this.tokenIndex].type === t1 &&
128 this.tokens[this.tokenIndex + 1].type === t2 &&
129 this.tokens[this.tokenIndex + 2].type === t3 &&
130 this.tokens[this.tokenIndex + 3].type === t4 &&
131 this.tokens[this.tokenIndex + 4].type === t5
132 );
133 }
134
135 matchesContextual(contextualKeyword) {
136 return this.matchesContextualAtIndex(this.tokenIndex, contextualKeyword);
137 }
138
139 matchesContextIdAndLabel(type, contextId) {
140 return this.matches1(type) && this.currentToken().contextId === contextId;
141 }
142
143 previousWhitespaceAndComments() {
144 let whitespaceAndComments = this.code.slice(
145 this.tokenIndex > 0 ? this.tokens[this.tokenIndex - 1].end : 0,
146 this.tokenIndex < this.tokens.length ? this.tokens[this.tokenIndex].start : this.code.length,
147 );
148 if (this.isFlowEnabled) {
149 whitespaceAndComments = whitespaceAndComments.replace(/@flow/g, "");
150 }
151 return whitespaceAndComments;
152 }
153
154 replaceToken(newCode) {
155 this.resultCode += this.previousWhitespaceAndComments();
156 this.appendTokenPrefix();
157 this.resultCode += newCode;
158 this.appendTokenSuffix();
159 this.tokenIndex++;
160 }
161
162 replaceTokenTrimmingLeftWhitespace(newCode) {
163 this.resultCode += this.previousWhitespaceAndComments().replace(/[^\r\n]/g, "");
164 this.appendTokenPrefix();
165 this.resultCode += newCode;
166 this.appendTokenSuffix();
167 this.tokenIndex++;
168 }
169
170 removeInitialToken() {
171 this.replaceToken("");
172 }
173
174 removeToken() {
175 this.replaceTokenTrimmingLeftWhitespace("");
176 }
177
178 copyExpectedToken(tokenType) {
179 if (this.tokens[this.tokenIndex].type !== tokenType) {
180 throw new Error(`Expected token ${tokenType}`);
181 }
182 this.copyToken();
183 }
184
185 copyToken() {
186 this.resultCode += this.previousWhitespaceAndComments();
187 this.appendTokenPrefix();
188 this.resultCode += this.code.slice(
189 this.tokens[this.tokenIndex].start,
190 this.tokens[this.tokenIndex].end,
191 );
192 this.appendTokenSuffix();
193 this.tokenIndex++;
194 }
195
196 copyTokenWithPrefix(prefix) {
197 this.resultCode += this.previousWhitespaceAndComments();
198 this.appendTokenPrefix();
199 this.resultCode += prefix;
200 this.resultCode += this.code.slice(
201 this.tokens[this.tokenIndex].start,
202 this.tokens[this.tokenIndex].end,
203 );
204 this.appendTokenSuffix();
205 this.tokenIndex++;
206 }
207
208 appendTokenPrefix() {
209 const token = this.currentToken();
210 if (token.numNullishCoalesceStarts || token.isOptionalChainStart) {
211 token.isAsyncOperation = isAsyncOperation(this);
212 }
213 if (token.numNullishCoalesceStarts) {
214 for (let i = 0; i < token.numNullishCoalesceStarts; i++) {
215 if (token.isAsyncOperation) {
216 this.resultCode += "await ";
217 this.resultCode += this.helperManager.getHelperName("asyncNullishCoalesce");
218 } else {
219 this.resultCode += this.helperManager.getHelperName("nullishCoalesce");
220 }
221 this.resultCode += "(";
222 }
223 }
224 if (token.isOptionalChainStart) {
225 if (token.isAsyncOperation) {
226 this.resultCode += "await ";
227 }
228 if (this.tokenIndex > 0 && this.tokenAtRelativeIndex(-1).type === tt._delete) {
229 if (token.isAsyncOperation) {
230 this.resultCode += this.helperManager.getHelperName("asyncOptionalChainDelete");
231 } else {
232 this.resultCode += this.helperManager.getHelperName("optionalChainDelete");
233 }
234 } else if (token.isAsyncOperation) {
235 this.resultCode += this.helperManager.getHelperName("asyncOptionalChain");
236 } else {
237 this.resultCode += this.helperManager.getHelperName("optionalChain");
238 }
239 this.resultCode += "([";
240 }
241 }
242
243 appendTokenSuffix() {
244 const token = this.currentToken();
245 if (token.isOptionalChainEnd) {
246 this.resultCode += "])";
247 }
248 if (token.numNullishCoalesceEnds) {
249 for (let i = 0; i < token.numNullishCoalesceEnds; i++) {
250 this.resultCode += "))";
251 }
252 }
253 }
254
255 appendCode(code) {
256 this.resultCode += code;
257 }
258
259 currentToken() {
260 return this.tokens[this.tokenIndex];
261 }
262
263 currentTokenCode() {
264 const token = this.currentToken();
265 return this.code.slice(token.start, token.end);
266 }
267
268 tokenAtRelativeIndex(relativeIndex) {
269 return this.tokens[this.tokenIndex + relativeIndex];
270 }
271
272 currentIndex() {
273 return this.tokenIndex;
274 }
275
276 /**
277 * Move to the next token. Only suitable in preprocessing steps. When
278 * generating new code, you should use copyToken or removeToken.
279 */
280 nextToken() {
281 if (this.tokenIndex === this.tokens.length) {
282 throw new Error("Unexpectedly reached end of input.");
283 }
284 this.tokenIndex++;
285 }
286
287 previousToken() {
288 this.tokenIndex--;
289 }
290
291 finish() {
292 if (this.tokenIndex !== this.tokens.length) {
293 throw new Error("Tried to finish processing tokens before reaching the end.");
294 }
295 this.resultCode += this.previousWhitespaceAndComments();
296 return this.resultCode;
297 }
298
299 isAtEnd() {
300 return this.tokenIndex === this.tokens.length;
301 }
302}