UNPKG

2.93 kBTypeScriptView Raw
1/**
2 * Splits an input string into lexical tokens, i.e. smaller strings that are
3 * easily identifiable by `tokens.tokenType()`.
4 *
5 * Lexing starts always in a "stream" context. Incomplete input may be buffered
6 * until a complete token can be emitted.
7 *
8 * In addition to slices of the original input, the following control characters
9 * may also be emitted:
10 *
11 * - `\x02` (Start of Text): A document starts with the next token
12 * - `\x18` (Cancel): Unexpected end of flow-mode (indicates an error)
13 * - `\x1f` (Unit Separator): Next token is a scalar value
14 * - `\u{FEFF}` (Byte order mark): Emitted separately outside documents
15 */
16export declare class Lexer {
17 /**
18 * Flag indicating whether the end of the current buffer marks the end of
19 * all input
20 */
21 private atEnd;
22 /**
23 * Explicit indent set in block scalar header, as an offset from the current
24 * minimum indent, so e.g. set to 1 from a header `|2+`. Set to -1 if not
25 * explicitly set.
26 */
27 private blockScalarIndent;
28 /**
29 * Block scalars that include a + (keep) chomping indicator in their header
30 * include trailing empty lines, which are otherwise excluded from the
31 * scalar's contents.
32 */
33 private blockScalarKeep;
34 /** Current input */
35 private buffer;
36 /**
37 * Flag noting whether the map value indicator : can immediately follow this
38 * node within a flow context.
39 */
40 private flowKey;
41 /** Count of surrounding flow collection levels. */
42 private flowLevel;
43 /**
44 * Minimum level of indentation required for next lines to be parsed as a
45 * part of the current scalar value.
46 */
47 private indentNext;
48 /** Indentation level of the current line. */
49 private indentValue;
50 /** Position of the next \n character. */
51 private lineEndPos;
52 /** Stores the state of the lexer if reaching the end of incpomplete input */
53 private next;
54 /** A pointer to `buffer`; the current position of the lexer. */
55 private pos;
56 /**
57 * Generate YAML tokens from the `source` string. If `incomplete`,
58 * a part of the last line may be left as a buffer for the next call.
59 *
60 * @returns A generator of lexical tokens
61 */
62 lex(source: string, incomplete?: boolean): Generator<string, void, any>;
63 private atLineEnd;
64 private charAt;
65 private continueScalar;
66 private getLine;
67 private hasChars;
68 private setNext;
69 private peek;
70 private parseNext;
71 private parseStream;
72 private parseLineStart;
73 private parseBlockStart;
74 private parseDocument;
75 private parseFlowCollection;
76 private parseQuotedScalar;
77 private parseBlockScalarHeader;
78 private parseBlockScalar;
79 private parsePlainScalar;
80 private pushCount;
81 private pushToIndex;
82 private pushIndicators;
83 private pushTag;
84 private pushNewline;
85 private pushSpaces;
86 private pushUntil;
87}