1 | /*!
|
2 | * Copyright 2016 The ANTLR Project. All rights reserved.
|
3 | * Licensed under the BSD-3-Clause license. See LICENSE file in the project root for license information.
|
4 | */
|
5 | import { CharStream } from "./CharStream";
|
6 | import { Token } from "./Token";
|
7 | import { TokenFactory } from "./TokenFactory";
|
8 | /**
|
9 | * A source of tokens must provide a sequence of tokens via {@link #nextToken()}
|
10 | * and also must reveal it's source of characters; {@link CommonToken}'s text is
|
11 | * computed from a {@link CharStream}; it only store indices into the char
|
12 | * stream.
|
13 | *
|
14 | * Errors from the lexer are never passed to the parser. Either you want to keep
|
15 | * going or you do not upon token recognition error. If you do not want to
|
16 | * continue lexing then you do not want to continue parsing. Just throw an
|
17 | * exception not under {@link RecognitionException} and Java will naturally toss
|
18 | * you all the way out of the recognizers. If you want to continue lexing then
|
19 | * you should not throw an exception to the parser--it has already requested a
|
20 | * token. Keep lexing until you get a valid one. Just report errors and keep
|
21 | * going, looking for a valid token.
|
22 | */
|
23 | export interface TokenSource {
|
24 | /**
|
25 | * Return a {@link Token} object from your input stream (usually a
|
26 | * {@link CharStream}). Do not fail/return upon lexing error; keep chewing
|
27 | * on the characters until you get a good one; errors are not passed through
|
28 | * to the parser.
|
29 | */
|
30 | nextToken(): Token;
|
31 | /**
|
32 | * Get the line number for the current position in the input stream. The
|
33 | * first line in the input is line 1.
|
34 | *
|
35 | * @returns The line number for the current position in the input stream, or
|
36 | * 0 if the current token source does not track line numbers.
|
37 | */
|
38 | readonly line: number;
|
39 | /**
|
40 | * Get the index into the current line for the current position in the input
|
41 | * stream. The first character on a line has position 0.
|
42 | *
|
43 | * @returns The line number for the current position in the input stream, or
|
44 | * -1 if the current token source does not track character positions.
|
45 | */
|
46 | readonly charPositionInLine: number;
|
47 | /**
|
48 | * Get the {@link CharStream} from which this token source is currently
|
49 | * providing tokens.
|
50 | *
|
51 | * @returns The {@link CharStream} associated with the current position in
|
52 | * the input, or `undefined` if no input stream is available for the token
|
53 | * source.
|
54 | */
|
55 | readonly inputStream: CharStream | undefined;
|
56 | /**
|
57 | * Gets the name of the underlying input source. This method returns a
|
58 | * non-undefined, non-empty string. If such a name is not known, this method
|
59 | * returns {@link IntStream#UNKNOWN_SOURCE_NAME}.
|
60 | */
|
61 | readonly sourceName: string;
|
62 | /**
|
63 | * Gets or sets the `TokenFactory` this token source is currently using for
|
64 | * creating `Token` objects from the input.
|
65 | */
|
66 | tokenFactory: TokenFactory;
|
67 | }
|