| 1 | /*!
|
| 2 | * Copyright 2016 The ANTLR Project. All rights reserved.
|
| 3 | * Licensed under the BSD-3-Clause license. See LICENSE file in the project root for license information.
|
| 4 | */
|
| 5 | import { CharStream } from "./CharStream";
|
| 6 | import { TokenSource } from "./TokenSource";
|
| 7 | /** A token has properties: text, type, line, character position in the line
|
| 8 | * (so we can ignore tabs), token channel, index, and source from which
|
| 9 | * we obtained this token.
|
| 10 | */
|
| 11 | export interface Token {
|
| 12 | /**
|
| 13 | * Get the text of the token.
|
| 14 | */
|
| 15 | readonly text: string | undefined;
|
| 16 | /** Get the token type of the token */
|
| 17 | readonly type: number;
|
| 18 | /** The line number on which the 1st character of this token was matched,
|
| 19 | * line=1..n
|
| 20 | */
|
| 21 | readonly line: number;
|
| 22 | /** The index of the first character of this token relative to the
|
| 23 | * beginning of the line at which it occurs, 0..n-1
|
| 24 | */
|
| 25 | readonly charPositionInLine: number;
|
| 26 | /** Return the channel this token. Each token can arrive at the parser
|
| 27 | * on a different channel, but the parser only "tunes" to a single channel.
|
| 28 | * The parser ignores everything not on DEFAULT_CHANNEL.
|
| 29 | */
|
| 30 | readonly channel: number;
|
| 31 | /** An index from 0..n-1 of the token object in the input stream.
|
| 32 | * This must be valid in order to print token streams and
|
| 33 | * use TokenRewriteStream.
|
| 34 | *
|
| 35 | * Return -1 to indicate that this token was conjured up since
|
| 36 | * it doesn't have a valid index.
|
| 37 | */
|
| 38 | readonly tokenIndex: number;
|
| 39 | /** The starting character index of the token
|
| 40 | * This method is optional; return -1 if not implemented.
|
| 41 | */
|
| 42 | readonly startIndex: number;
|
| 43 | /** The last character index of the token.
|
| 44 | * This method is optional; return -1 if not implemented.
|
| 45 | */
|
| 46 | readonly stopIndex: number;
|
| 47 | /** Gets the {@link TokenSource} which created this token.
|
| 48 | */
|
| 49 | readonly tokenSource: TokenSource | undefined;
|
| 50 | /**
|
| 51 | * Gets the {@link CharStream} from which this token was derived.
|
| 52 | */
|
| 53 | readonly inputStream: CharStream | undefined;
|
| 54 | }
|
| 55 | export declare namespace Token {
|
| 56 | const INVALID_TYPE: number;
|
| 57 | /** During lookahead operations, this "token" signifies we hit rule end ATN state
|
| 58 | * and did not follow it despite needing to.
|
| 59 | */
|
| 60 | const EPSILON: number;
|
| 61 | const MIN_USER_TOKEN_TYPE: number;
|
| 62 | const EOF: number;
|
| 63 | /** All tokens go to the parser (unless skip() is called in that rule)
|
| 64 | * on a particular "channel". The parser tunes to a particular channel
|
| 65 | * so that whitespace etc... can go to the parser on a "hidden" channel.
|
| 66 | */
|
| 67 | const DEFAULT_CHANNEL: number;
|
| 68 | /** Anything on different channel than DEFAULT_CHANNEL is not parsed
|
| 69 | * by parser.
|
| 70 | */
|
| 71 | const HIDDEN_CHANNEL: number;
|
| 72 | /**
|
| 73 | * This is the minimum constant value which can be assigned to a
|
| 74 | * user-defined token channel.
|
| 75 | *
|
| 76 | * The non-negative numbers less than {@link #MIN_USER_CHANNEL_VALUE} are
|
| 77 | * assigned to the predefined channels {@link #DEFAULT_CHANNEL} and
|
| 78 | * {@link #HIDDEN_CHANNEL}.
|
| 79 | *
|
| 80 | * @see `Token.channel`
|
| 81 | */
|
| 82 | const MIN_USER_CHANNEL_VALUE: number;
|
| 83 | }
|