1 | /*!
|
2 | * Copyright 2016 The ANTLR Project. All rights reserved.
|
3 | * Licensed under the BSD-3-Clause license. See LICENSE file in the project root for license information.
|
4 | */
|
5 | import { CharStream } from "./CharStream";
|
6 | import { TokenSource } from "./TokenSource";
|
7 | /** A token has properties: text, type, line, character position in the line
|
8 | * (so we can ignore tabs), token channel, index, and source from which
|
9 | * we obtained this token.
|
10 | */
|
11 | export interface Token {
|
12 | /**
|
13 | * Get the text of the token.
|
14 | */
|
15 | readonly text: string | undefined;
|
16 | /** Get the token type of the token */
|
17 | readonly type: number;
|
18 | /** The line number on which the 1st character of this token was matched,
|
19 | * line=1..n
|
20 | */
|
21 | readonly line: number;
|
22 | /** The index of the first character of this token relative to the
|
23 | * beginning of the line at which it occurs, 0..n-1
|
24 | */
|
25 | readonly charPositionInLine: number;
|
26 | /** Return the channel this token. Each token can arrive at the parser
|
27 | * on a different channel, but the parser only "tunes" to a single channel.
|
28 | * The parser ignores everything not on DEFAULT_CHANNEL.
|
29 | */
|
30 | readonly channel: number;
|
31 | /** An index from 0..n-1 of the token object in the input stream.
|
32 | * This must be valid in order to print token streams and
|
33 | * use TokenRewriteStream.
|
34 | *
|
35 | * Return -1 to indicate that this token was conjured up since
|
36 | * it doesn't have a valid index.
|
37 | */
|
38 | readonly tokenIndex: number;
|
39 | /** The starting character index of the token
|
40 | * This method is optional; return -1 if not implemented.
|
41 | */
|
42 | readonly startIndex: number;
|
43 | /** The last character index of the token.
|
44 | * This method is optional; return -1 if not implemented.
|
45 | */
|
46 | readonly stopIndex: number;
|
47 | /** Gets the {@link TokenSource} which created this token.
|
48 | */
|
49 | readonly tokenSource: TokenSource | undefined;
|
50 | /**
|
51 | * Gets the {@link CharStream} from which this token was derived.
|
52 | */
|
53 | readonly inputStream: CharStream | undefined;
|
54 | }
|
55 | export declare namespace Token {
|
56 | const INVALID_TYPE: number;
|
57 | /** During lookahead operations, this "token" signifies we hit rule end ATN state
|
58 | * and did not follow it despite needing to.
|
59 | */
|
60 | const EPSILON: number;
|
61 | const MIN_USER_TOKEN_TYPE: number;
|
62 | const EOF: number;
|
63 | /** All tokens go to the parser (unless skip() is called in that rule)
|
64 | * on a particular "channel". The parser tunes to a particular channel
|
65 | * so that whitespace etc... can go to the parser on a "hidden" channel.
|
66 | */
|
67 | const DEFAULT_CHANNEL: number;
|
68 | /** Anything on different channel than DEFAULT_CHANNEL is not parsed
|
69 | * by parser.
|
70 | */
|
71 | const HIDDEN_CHANNEL: number;
|
72 | /**
|
73 | * This is the minimum constant value which can be assigned to a
|
74 | * user-defined token channel.
|
75 | *
|
76 | * The non-negative numbers less than {@link #MIN_USER_CHANNEL_VALUE} are
|
77 | * assigned to the predefined channels {@link #DEFAULT_CHANNEL} and
|
78 | * {@link #HIDDEN_CHANNEL}.
|
79 | *
|
80 | * @see `Token.channel`
|
81 | */
|
82 | const MIN_USER_CHANNEL_VALUE: number;
|
83 | }
|