All files / mframejs/binding/ast tokenize.ts

1
2
3
4
5
6
7
8
9
10
11
12
13
14
29x
 
 
 
 
 
 
29x
326x
326x
 
326x
 
 
import { Tokenizer } from './tokenizer';
 
 
 
/**
 * Will tokenize string (only give it content of ${})
 */
export const tokenize = function (str: string) {
    const tokenizer = new Tokenizer(str);
    const tokens = tokenizer.start();
 
    return tokens;
};