UNPKG

1.29 kBTypeScriptView Raw
1import { Token } from './ast';
2import { Source } from './source';
3import { TokenKindEnum } from './tokenKind';
4
5/**
6 * Given a Source object, this returns a Lexer for that source.
7 * A Lexer is a stateful stream generator in that every time
8 * it is advanced, it returns the next token in the Source. Assuming the
9 * source lexes, the final Token emitted by the lexer will be of kind
10 * EOF, after which the lexer will repeatedly return the same EOF token
11 * whenever called.
12 */
13export class Lexer {
14 source: Source;
15
16 /**
17 * The previously focused non-ignored token.
18 */
19 lastToken: Token;
20
21 /**
22 * The currently focused non-ignored token.
23 */
24 token: Token;
25
26 /**
27 * The (1-indexed) line containing the current token.
28 */
29 line: number;
30
31 /**
32 * The character offset at which the current line begins.
33 */
34 lineStart: number;
35
36 constructor(source: Source);
37
38 /**
39 * Advances the token stream to the next non-ignored token.
40 */
41 advance(): Token;
42
43 /**
44 * Looks ahead and returns the next non-ignored token, but does not change
45 * the state of Lexer.
46 */
47 lookahead(): Token;
48}
49
50/**
51 * @internal
52 */
53export function isPunctuatorToken(token: Token): boolean;
54
55/**
56 * @internal
57 */
58export function isPunctuatorTokenKind(kind: TokenKindEnum): boolean;