| 1 | /**
|
| 2 | * Token type.
|
| 3 | */
|
| 4 | export declare type TokenType = 'ident' | '(' | ')' | ',' | '=' | '*' | 'function' | 'class' | 'EOF';
|
| 5 | /**
|
| 6 | * Lexer Token.
|
| 7 | */
|
| 8 | export interface Token {
|
| 9 | type: TokenType;
|
| 10 | value?: string;
|
| 11 | }
|
| 12 | /**
|
| 13 | * Flags that can be passed to the tokenizer to toggle certain things.
|
| 14 | */
|
| 15 | export declare const enum TokenizerFlags {
|
| 16 | None = 0,
|
| 17 | /**
|
| 18 | * If this is set, the tokenizer will not attempt to be smart about skipping expressions.
|
| 19 | */
|
| 20 | Dumb = 1
|
| 21 | }
|
| 22 | /**
|
| 23 | * Creates a tokenizer for the specified source.
|
| 24 | *
|
| 25 | * @param source
|
| 26 | */
|
| 27 | export declare function createTokenizer(source: string): {
|
| 28 | next: (nextFlags?: TokenizerFlags) => Token;
|
| 29 | done: () => boolean;
|
| 30 | };
|