import type { Position } from 'unist';
export declare const Token: {
    readonly Word: "Word";
    readonly WhiteSpace: "WhiteSpace";
    readonly Comment: "Comment";
    readonly NewLine: "NewLine";
};
export interface Token {
    type: keyof typeof Token;
    value: string;
    position: DeepNonNullish<Position>;
}
export declare function tokenize(input: string): Generator<Token, void, undefined>;
/** Recursively makes all properties of `T` nullable and optional. */
type DeepNonNullish<T> = T extends object ? {
    [K in keyof T]-?: DeepNonNullish<T[K]>;
} : NonNullable<T>;
export {};