UNPKG

1.34 kBTypeScriptView Raw
1import { syntaxError } from '../error';
2import { Token } from './ast';
3import { Source } from './source';
4
5/**
6 * Given a Source object, this returns a Lexer for that source.
7 * A Lexer is a stateful stream generator in that every time
8 * it is advanced, it returns the next token in the Source. Assuming the
9 * source lexes, the final Token emitted by the lexer will be of kind
10 * EOF, after which the lexer will repeatedly return the same EOF token
11 * whenever called.
12 */
13export function createLexer<TOptions>(
14 source: Source,
15 options: TOptions,
16): Lexer<TOptions>;
17
18/**
19 * The return type of createLexer.
20 */
21export interface Lexer<TOptions> {
22 source: Source;
23 options: TOptions;
24
25 /**
26 * The previously focused non-ignored token.
27 */
28 lastToken: Token;
29
30 /**
31 * The currently focused non-ignored token.
32 */
33 token: Token;
34
35 /**
36 * The (1-indexed) line containing the current token.
37 */
38 line: number;
39
40 /**
41 * The character offset at which the current line begins.
42 */
43 lineStart: number;
44
45 /**
46 * Advances the token stream to the next non-ignored token.
47 */
48 advance(): Token;
49
50 /**
51 * Looks ahead and returns the next non-ignored token, but does not change
52 * the Lexer's state.
53 */
54 lookahead(): Token;
55}
56
57/**
58 * @internal
59 */
60export function isPunctuatorToken(token: Token): boolean;