| 1 | /*!
|
| 2 | * Copyright 2016 The ANTLR Project. All rights reserved.
|
| 3 | * Licensed under the BSD-3-Clause license. See LICENSE file in the project root for license information.
|
| 4 | */
|
| 5 | import { CharStream } from "./CharStream";
|
| 6 | import { Token } from "./Token";
|
| 7 | import { TokenFactory } from "./TokenFactory";
|
| 8 | /**
|
| 9 | * A source of tokens must provide a sequence of tokens via {@link #nextToken()}
|
| 10 | * and also must reveal it's source of characters; {@link CommonToken}'s text is
|
| 11 | * computed from a {@link CharStream}; it only store indices into the char
|
| 12 | * stream.
|
| 13 | *
|
| 14 | * Errors from the lexer are never passed to the parser. Either you want to keep
|
| 15 | * going or you do not upon token recognition error. If you do not want to
|
| 16 | * continue lexing then you do not want to continue parsing. Just throw an
|
| 17 | * exception not under {@link RecognitionException} and Java will naturally toss
|
| 18 | * you all the way out of the recognizers. If you want to continue lexing then
|
| 19 | * you should not throw an exception to the parser--it has already requested a
|
| 20 | * token. Keep lexing until you get a valid one. Just report errors and keep
|
| 21 | * going, looking for a valid token.
|
| 22 | */
|
| 23 | export interface TokenSource {
|
| 24 | /**
|
| 25 | * Return a {@link Token} object from your input stream (usually a
|
| 26 | * {@link CharStream}). Do not fail/return upon lexing error; keep chewing
|
| 27 | * on the characters until you get a good one; errors are not passed through
|
| 28 | * to the parser.
|
| 29 | */
|
| 30 | nextToken(): Token;
|
| 31 | /**
|
| 32 | * Get the line number for the current position in the input stream. The
|
| 33 | * first line in the input is line 1.
|
| 34 | *
|
| 35 | * @returns The line number for the current position in the input stream, or
|
| 36 | * 0 if the current token source does not track line numbers.
|
| 37 | */
|
| 38 | readonly line: number;
|
| 39 | /**
|
| 40 | * Get the index into the current line for the current position in the input
|
| 41 | * stream. The first character on a line has position 0.
|
| 42 | *
|
| 43 | * @returns The line number for the current position in the input stream, or
|
| 44 | * -1 if the current token source does not track character positions.
|
| 45 | */
|
| 46 | readonly charPositionInLine: number;
|
| 47 | /**
|
| 48 | * Get the {@link CharStream} from which this token source is currently
|
| 49 | * providing tokens.
|
| 50 | *
|
| 51 | * @returns The {@link CharStream} associated with the current position in
|
| 52 | * the input, or `undefined` if no input stream is available for the token
|
| 53 | * source.
|
| 54 | */
|
| 55 | readonly inputStream: CharStream | undefined;
|
| 56 | /**
|
| 57 | * Gets the name of the underlying input source. This method returns a
|
| 58 | * non-undefined, non-empty string. If such a name is not known, this method
|
| 59 | * returns {@link IntStream#UNKNOWN_SOURCE_NAME}.
|
| 60 | */
|
| 61 | readonly sourceName: string;
|
| 62 | /**
|
| 63 | * Gets or sets the `TokenFactory` this token source is currently using for
|
| 64 | * creating `Token` objects from the input.
|
| 65 | */
|
| 66 | tokenFactory: TokenFactory;
|
| 67 | }
|