declare type TokenMatcher = {
    pattern: string;
} | {
    id: string;
    pattern: string | RegExp;
};
interface NormalizedMatcher {
    id: string;
    pattern: RegExp;
}
export interface TokenMatchResult extends NormalizedMatcher {
    value: string;
}
export declare class Tokenizer {
    private matchAll;
    private matchers;
    constructor(matchers: TokenMatcher[]);
    tokenize(str: string): TokenMatchResult[];
}
export {};