import { TokenizerOptions, Token } from './types';
export default function tokenize(input: string, options?: TokenizerOptions): Token[];