1 |
|
2 | const {TokenTypeData, Token, Expr} = require('./lang');
|
3 | const NaiveCompiler = require('./naive-compiler');
|
4 | const _ = require('lodash');
|
5 | const fs = require('fs');
|
6 | const path = require('path');
|
7 |
|
8 | const babelParser = require('@babel/parser');
|
9 | const walk = require('babylon-walk');
|
10 | const generate = require('babel-generator');
|
11 | const optLibrary = require('./templates/optimizing').library.toString();
|
12 |
|
13 | const verbs = Object.keys(TokenTypeData)
|
14 | .filter(token => !TokenTypeData[token].nonVerb)
|
15 | .filter(token => token !== 'abstract')
|
16 | .filter(token => token !== 'cond');
|
17 | const lazyVerbs = new Set(['and', 'or', 'ternary']);
|
18 | const eagerVerbs = verbs.filter(token => !lazyVerbs.has(token));
|
19 | const verbsLazyFirst = Array.from(lazyVerbs).concat(eagerVerbs);
|
20 | const verbsSet = new Set(verbsLazyFirst);
|
21 | const nonVerbs = Object.keys(TokenTypeData).filter(token => TokenTypeData[token].nonVerb);
|
22 |
|
23 | const valueTypes = ['numberInline', 'booleanInline', 'stringRef', 'numberRef', 'expressionRef', 'condRef'];
|
24 |
|
25 | const setterTypes = ['setter', 'splice', 'push'];
|
26 |
|
27 | const defineEnum = (name, values) => `
|
28 | ${values.map((key, index) => `module.exports.$${key} = ${index};`).join('\n')}
|
29 | module.exports.${name}Count = ${values.length};
|
30 | module.exports.${name} = {
|
31 | ${values.map((key, index) => ` $${key}: ${index}`).join(',\n')}
|
32 | };
|
33 |
|
34 | `;
|
35 |
|
36 | const enums = `
|
37 | ${defineEnum('Verbs', verbsLazyFirst)}
|
38 | ${defineEnum('nonVerbs', valueTypes.concat(nonVerbs))}
|
39 | ${defineEnum('setterTypes', setterTypes)}
|
40 |
|
41 | // Values are uint32
|
42 | // VVVVVVVVVVVVVVVVVVVVVVVVVVVVVTTT
|
43 | // TTT - signifies type
|
44 | // 000 - number inline
|
45 | // 001 - boolean inline
|
46 | // 010 - string ref
|
47 | // 011 - number ref
|
48 | // 100 - token
|
49 | // 101 - expression ref
|
50 | // the rest of the bits are the value 2^29 possible values
|
51 |
|
52 | // table Expression {
|
53 | // token: Verbs;
|
54 | // values: [uint32] (required);
|
55 | // }
|
56 |
|
57 | // table Bytecode {
|
58 | // topLevels:[uint32] (required);
|
59 | // topLevelsNames:[uint32] (required);
|
60 | // expressions: [Expression] (required);
|
61 | // constant_numbers:[float64] (required);
|
62 | // constant_strings:[string] (required);
|
63 | // }
|
64 |
|
65 | // root_type Bytecode;
|
66 | //
|
67 |
|
68 | `;
|
69 | fs.writeFileSync(path.join(__dirname, '..', 'bytecode', 'bytecode-enums.js'), enums);
|
70 |
|
71 | const verbsNotAutoGenerated = new Set(['recur', 'func', 'array', 'object', 'recursiveMapValues', 'recursiveMap']);
|
72 |
|
73 | const naiveCompiler = new NaiveCompiler({}, {});
|
74 | const helperFunctions = eagerVerbs
|
75 | .map(verb => {
|
76 | if (
|
77 | verbsNotAutoGenerated.has(verb) ||
|
78 | TokenTypeData[verb].len[0] !== TokenTypeData[verb].len[1] ||
|
79 | TokenTypeData[verb].collectionVerb
|
80 | ) {
|
81 | return `// ${verb} skipped`;
|
82 | }
|
83 | const args = new Array(TokenTypeData[verb].len[0] - 1).fill().map((_, i) => new Token(`arg${i}`));
|
84 | const expr = Expr(new Token(verb), ...args);
|
85 | return `
|
86 | module.exports.$${verb} = function $${verb}($offset, $len) {
|
87 | ${args
|
88 | .map(naiveCompiler.generateExpr)
|
89 | .map(
|
90 | (id, index) =>
|
91 | ` this.processValue(this.$expressions[++$offset])
|
92 | const ${id} = this.$stack.pop();`
|
93 | )
|
94 | .join('\n')}
|
95 | this.$stack.push(${naiveCompiler.generateExpr(expr)});
|
96 | }`;
|
97 | })
|
98 | .join('\n');
|
99 |
|
100 | const ast = babelParser.parse(optLibrary, {plugins: []});
|
101 | const statements = ast.program.body[0].body.body;
|
102 | const functions = statements.filter(t => t.type === 'FunctionDeclaration');
|
103 | const functionNames = new Set(functions.map(t => t.id.name));
|
104 | const constFuncs = statements
|
105 | .filter(t => t.type === 'VariableDeclaration')
|
106 | .filter(t => t.declarations[0].init.type === 'ArrowFunctionExpression');
|
107 |
|
108 | const constFuncsNames = new Set(constFuncs.map(t => t.declarations[0].id.name));
|
109 |
|
110 | const constFuncSources = Array.from(constFuncs)
|
111 | .filter(f => f.declarations[0].id.name !== 'recursiveCacheFunc')
|
112 | .map(f => generate.default(f).code)
|
113 | .join('\n');
|
114 |
|
115 | const constValues = statements
|
116 | .filter(t => t.type === 'VariableDeclaration')
|
117 | .filter(t => t.declarations[0].init.type !== 'ArrowFunctionExpression');
|
118 |
|
119 | const constValuesNames = new Set(
|
120 | constValues.map(t => t.declarations[0].id.name).concat(['$res', '$funcLib', '$funcLibRaw'])
|
121 | );
|
122 |
|
123 | function rewriteAncestorBlockStatement(ancestors, deleteCount, ...newItems) {
|
124 | const innerMostBlockStatementIndex = _.findLastIndex(ancestors, {type: 'BlockStatement'});
|
125 | const block = ancestors[innerMostBlockStatementIndex];
|
126 | const currentIndex = _.findIndex(block.body, ancestors[innerMostBlockStatementIndex + 1]);
|
127 | if (deleteCount > 0 && newItems.length) {
|
128 | ancestors[innerMostBlockStatementIndex + 1] = newItems[0];
|
129 | }
|
130 | return block.body.splice(currentIndex, deleteCount, ...newItems);
|
131 | }
|
132 |
|
133 | const visitorFuncDeclStatements = {
|
134 | ReturnStatement(node, state, ancestors) {
|
135 | if (ancestors.length > 3) {
|
136 | return;
|
137 | }
|
138 | Object.assign(node, {
|
139 | arguments: [node.argument],
|
140 | type: 'CallExpression',
|
141 | callee: {
|
142 | type: 'MemberExpression',
|
143 | object: {
|
144 | type: 'MemberExpression',
|
145 | object: {
|
146 | type: 'ThisExpression'
|
147 | },
|
148 | property: {
|
149 | type: 'Identifier',
|
150 | name: '$stack'
|
151 | },
|
152 | computed: false
|
153 | },
|
154 | property: {
|
155 | type: 'Identifier',
|
156 | name: 'push'
|
157 | },
|
158 | computed: false
|
159 | }
|
160 | });
|
161 | delete node.argument;
|
162 | },
|
163 | CallExpression(node, state, ancestors) {
|
164 | if (node.callee.name === 'getEmptyArray' || node.callee.name === 'getEmptyObject') {
|
165 | node.arguments = [
|
166 | {
|
167 | type: 'UnaryExpression',
|
168 | operator: '-',
|
169 | prefix: true,
|
170 | argument: {
|
171 | type: 'Identifier',
|
172 | name: '$offset'
|
173 | }
|
174 | }
|
175 | ];
|
176 | } else if (node.callee.name === 'initOutput') {
|
177 | const hasCache = node.arguments[4].name !== 'nullFunc';
|
178 | node.arguments.splice(0, 2);
|
179 | node.arguments[0] = {
|
180 | type: 'BinaryExpression',
|
181 | left: {
|
182 | type: 'Identifier',
|
183 | name: '$offset'
|
184 | },
|
185 | operator: '-',
|
186 | right: {
|
187 | type: 'Identifier',
|
188 | name: '$length'
|
189 | }
|
190 | };
|
191 | } else if (node.callee.name === 'func') {
|
192 | const arg = node.arguments[1];
|
193 | node.callee = {
|
194 | type: 'MemberExpression',
|
195 | object: {
|
196 | type: 'MemberExpression',
|
197 | object: {
|
198 | type: 'ThisExpression'
|
199 | },
|
200 | property: {
|
201 | type: 'Identifier',
|
202 | name: '$stack'
|
203 | },
|
204 | computed: false
|
205 | },
|
206 | property: {
|
207 | type: 'Identifier',
|
208 | name: 'pop'
|
209 | },
|
210 | computed: false
|
211 | };
|
212 | node.arguments = [];
|
213 | rewriteAncestorBlockStatement(
|
214 | ancestors,
|
215 | 0,
|
216 | {
|
217 | type: 'ExpressionStatement',
|
218 | expression: {
|
219 | type: 'CallExpression',
|
220 | callee: {
|
221 | type: 'MemberExpression',
|
222 | object: {
|
223 | type: 'MemberExpression',
|
224 | object: {
|
225 | type: 'ThisExpression'
|
226 | },
|
227 | property: {
|
228 | type: 'Identifier',
|
229 | name: '$keys'
|
230 | },
|
231 | computed: false
|
232 | },
|
233 | property: {
|
234 | type: 'Identifier',
|
235 | name: 'push'
|
236 | },
|
237 | computed: false
|
238 | },
|
239 | arguments: [arg]
|
240 | }
|
241 | },
|
242 | {
|
243 | type: 'ExpressionStatement',
|
244 | expression: {
|
245 | type: 'CallExpression',
|
246 | callee: {
|
247 | type: 'MemberExpression',
|
248 | object: {
|
249 | type: 'ThisExpression'
|
250 | },
|
251 | property: {
|
252 | type: 'Identifier',
|
253 |
|
254 | name: 'collectionFunction'
|
255 | },
|
256 | computed: false
|
257 | },
|
258 | arguments: []
|
259 | }
|
260 | }
|
261 | );
|
262 | }
|
263 |
|
264 |
|
265 | }
|
266 | };
|
267 |
|
268 | const visitorsPointFunctionsToThis = {
|
269 | Identifier(node, state, ancestors) {
|
270 | if (
|
271 | constValuesNames.has(node.name) &&
|
272 | (ancestors[ancestors.length - 2].type !== 'MemberExpression' ||
|
273 | ancestors[ancestors.length - 2].object.type !== 'ThisExpression')
|
274 | ) {
|
275 | Object.assign(node, {
|
276 | type: 'MemberExpression',
|
277 | object: {
|
278 | type: 'ThisExpression'
|
279 | },
|
280 | property: {
|
281 | type: 'Identifier',
|
282 | name: node.name
|
283 | },
|
284 | computed: false
|
285 | });
|
286 | delete node.name;
|
287 | }
|
288 | },
|
289 | CallExpression(node, state, ancestors) {
|
290 | if (
|
291 | node.callee.name &&
|
292 | functionNames.has(node.callee.name) &&
|
293 | (ancestors[ancestors.length - 2].type !== 'MemberExpression' ||
|
294 | ancestors[ancestors.length - 2].object.type !== 'ThisExpression')
|
295 | ) {
|
296 | node.callee = {
|
297 | type: 'MemberExpression',
|
298 | object: {
|
299 | type: 'ThisExpression'
|
300 | },
|
301 | property: {
|
302 | type: 'Identifier',
|
303 | name: node.callee.name
|
304 | },
|
305 | computed: false
|
306 | };
|
307 | }
|
308 | }
|
309 | };
|
310 |
|
311 | const functionsById = functions.reduce((acc, f) => ({...acc, [f.id.name]: f}), {});
|
312 | const verbFunctions = Object.keys(functionsById).reduce((acc, name) => {
|
313 | if (verbsSet.has(name) || verbsSet.has(name.replace('Opt', ''))) {
|
314 | return {...acc, [verbsSet.has(name) ? name : name.replace('Opt', '')]: functionsById[name]};
|
315 | }
|
316 | return acc;
|
317 | }, {});
|
318 |
|
319 | const verbsIgnoredInOptimizing = new Set(['recur', 'func', 'recursiveMapValues', 'recursiveMap']);
|
320 |
|
321 | const snippets = _.mapValues(
|
322 | {
|
323 | srcPre: ($offset, $length) => {
|
324 | this.processValue(this.$expressions[++$offset]);
|
325 | },
|
326 | src: ($offset, $length) => {
|
327 | let src = this.$stack.pop();
|
328 | this.$collections.push(src);
|
329 | },
|
330 | srcEnd: ($offset, $length) => {
|
331 | this.$collections.pop();
|
332 | this.$currentSets.pop();
|
333 | },
|
334 | contextPre: ($offset, $length) => {
|
335 | if ($length === 3) {
|
336 | this.$stack.push(null);
|
337 | } else {
|
338 | this.processValue(this.$expressions[++$offset]);
|
339 | }
|
340 | },
|
341 | context: ($offset, $length) => {
|
342 | if ($length === 3) {
|
343 | this.$contexts.push(this.$stack.pop());
|
344 | } else {
|
345 | const contextArray = this.getEmptyArray(~$offset);
|
346 | if (contextArray.length) {
|
347 | this.setOnArray(contextArray, 0, this.$stack.pop(), false);
|
348 | } else {
|
349 | contextArray[0] = this.$stack.pop();
|
350 | }
|
351 | this.$contexts.push(contextArray);
|
352 | }
|
353 | },
|
354 | contextEnd: ($offset, $length) => {
|
355 | this.$contexts.pop();
|
356 | },
|
357 | func: ($offset, $length) => {
|
358 |
|
359 | this.$functions.push(func);
|
360 | },
|
361 | funcPre: ($offset, $length) => {
|
362 | const func = this.$expressions[++$offset];
|
363 | },
|
364 | funcEnd: ($offset, $length) => {
|
365 | this.$functions.pop();
|
366 | },
|
367 | endPre: ($offset, $length) => {
|
368 | this.processValue(this.$expressions[++$offset]);
|
369 | },
|
370 | end: ($offset, $length) => {
|
371 | const end = this.$stack.pop();
|
372 | },
|
373 | startPre: ($offset, $length) => {
|
374 | if ($length > 2) {
|
375 | this.processValue(this.$expressions[++$offset]);
|
376 | } else {
|
377 | this.$stack.push(0);
|
378 | }
|
379 | },
|
380 | start: ($offset, $length) => {
|
381 | const start = this.$stack.pop();
|
382 | },
|
383 | stepPre: ($offset, $length) => {
|
384 | if ($length > 3) {
|
385 | this.processValue(this.$expressions[++$offset]);
|
386 | } else {
|
387 | this.$stack.push(1);
|
388 | }
|
389 | },
|
390 | step: ($offset, $length) => {
|
391 | const step = this.$stack.pop();
|
392 | },
|
393 | len: ($offset, $length) => {
|
394 | const len = $length - 1;
|
395 | },
|
396 | newValPre: ($offset, $length) => {
|
397 | const newVal = [];
|
398 | for (let i = 1; i < $length; i++) {
|
399 | this.processValue(this.$expressions[++$offset]);
|
400 | newVal.push(this.$stack.pop());
|
401 | }
|
402 | },
|
403 | keysListPre: ($offset, $length) => {
|
404 | let keysList = this.$globals.get($offset);
|
405 | if (!keysList) {
|
406 | keysList = [];
|
407 | for (let i = 1; i < $length; i += 2) {
|
408 | this.processValue(this.$expressions[$offset + i]);
|
409 | keysList.push(this.$stack.pop());
|
410 | }
|
411 | this.$globals.set($offset, keysList);
|
412 | }
|
413 | },
|
414 | valsListPre: ($offset, $length) => {
|
415 | const valsList = [];
|
416 | for (let i = 2; i < $length; i += 2) {
|
417 | this.processValue(this.$expressions[$offset + i]);
|
418 | valsList.push(this.$stack.pop());
|
419 | }
|
420 | }
|
421 | },
|
422 | f => {
|
423 | const snippetAst = babelParser.parse(f.toString(), {plugins: []});
|
424 | return snippetAst.program.body[0].expression.body.body;
|
425 | }
|
426 | );
|
427 |
|
428 | const extractedFuncs = Object.entries(verbFunctions)
|
429 | .map(([name, f]) => {
|
430 | if (verbsIgnoredInOptimizing.has(name)) {
|
431 | return `// ${name} skipped from optimizing`;
|
432 | }
|
433 | f.id.name = name;
|
434 | const paramsPre = _.flatten(
|
435 | f.params
|
436 | .map(t => `${t.name}Pre`)
|
437 | .map(t => snippets[t])
|
438 | .filter(Boolean)
|
439 | );
|
440 | const params = _.flatten(
|
441 | _.reverse(f.params
|
442 | .map(t => t.name)
|
443 | .map(t => snippets[t])
|
444 | .filter(Boolean)
|
445 | )
|
446 | );
|
447 | const paramEnds = _.flatten(
|
448 | f.params
|
449 | .map(t => `${t.name}End`)
|
450 | .map(t => snippets[t])
|
451 | .filter(Boolean)
|
452 | );
|
453 | walk.ancestor(f, visitorFuncDeclStatements, []);
|
454 | walk.ancestor(f, visitorsPointFunctionsToThis, []);
|
455 | f.body.body.splice(0, 0, ...paramsPre, ...params);
|
456 | f.body.body.push(...paramEnds);
|
457 |
|
458 |
|
459 | f.params = [
|
460 | {
|
461 | type: 'Identifier',
|
462 | name: '$offset'
|
463 | },
|
464 | {
|
465 | type: 'Identifier',
|
466 | name: '$length'
|
467 | }
|
468 | ];
|
469 | f = {
|
470 | type: 'ExpressionStatement',
|
471 | expression: {
|
472 | type: 'AssignmentExpression',
|
473 | operator: '=',
|
474 | left: {
|
475 | type: 'MemberExpression',
|
476 | object: {
|
477 | type: 'MemberExpression',
|
478 | object: {
|
479 | type: 'Identifier',
|
480 | name: 'module'
|
481 | },
|
482 | property: {
|
483 | type: 'Identifier',
|
484 | name: 'exports'
|
485 | },
|
486 | computed: false
|
487 | },
|
488 | property: {
|
489 | type: 'Identifier',
|
490 | name: `$${name}`
|
491 | },
|
492 | computed: false
|
493 | },
|
494 | right: f
|
495 | }
|
496 | };
|
497 | return generate.default(f).code;
|
498 | })
|
499 | .join('\n');
|
500 |
|
501 |
|
502 | const keepNonVerbFunctions = ['untrack', 'invalidate', 'setOnObject', 'deleteOnObject', 'setOnArray', 'truncateArray', 'track', 'trackPath', 'triggerInvalidations', 'initOutput', 'getEmptyArray', 'getEmptyObject', 'invalidatePath', 'set', 'splice']
|
503 | const nonVerbFunctions = keepNonVerbFunctions
|
504 | .map(name => functionsById[name])
|
505 | .map(f => {
|
506 | walk.ancestor(f, visitorsPointFunctionsToThis, []);
|
507 | return generate.default(f).code.replace(/^function /, '');
|
508 | });
|
509 |
|
510 | fs.writeFileSync(
|
511 | path.join(__dirname, '..', 'bytecode', 'bytecode-functions.js'),
|
512 | `
|
513 | ${helperFunctions}
|
514 | ${constFuncSources}
|
515 | ${extractedFuncs}
|
516 |
|
517 | /*
|
518 | // constantValues
|
519 | ${constValues
|
520 | .map(
|
521 | t =>
|
522 | generate.default({
|
523 | type: 'ExpressionStatement',
|
524 | expression: {
|
525 | type: 'AssignmentExpression',
|
526 | operator: '=',
|
527 | left: {
|
528 | type: 'MemberExpression',
|
529 | object: {
|
530 | type: 'ThisExpression'
|
531 | },
|
532 | property: {
|
533 | type: 'Identifier',
|
534 | name: t.declarations[0].id.name
|
535 | },
|
536 | computed: false
|
537 | },
|
538 | right: t.declarations[0].init
|
539 | }
|
540 | }).code
|
541 | )
|
542 | .join('\n')}
|
543 | ${nonVerbFunctions.join('\n')}
|
544 | */
|
545 |
|
546 | `
|
547 | );
|