import { Token, Tokenizer } from "../../types";
export declare class CoreTokenizer implements Tokenizer {
    private readonly _spaceExpression;
    private readonly _expressions;
    private _tokenizeSegment;
    private _tokenizeInput;
    tokenize(input: string): Array<Token>;
}