/**
 * @copyright Sister Software. All rights reserved.
 * @author Teffen Ellis, et al.
 * @license
 * See LICENSE file in the project root for full license information.
 */
/**
 * Default contractions used by the tokenizer pattern.
 * Note that order matters here, as the pattern will match the first contraction that matches.
 */
export declare const DEFAULT_CONTRACTIONS: readonly ["'s", "'t", "'re", "'ve", "'m", "'ll", "'d"];
/**
 * Default tokenizer rules used to build the tokenizer pattern.
 */
export declare const DEFAULT_TOKENIZER_RULES: readonly [" ?\\p{L}+", " ?\\p{N}+", " ?[^\\s\\p{L}\\p{N}]+", "\\s+(?!\\S)", "\\s+"];
/**
 * Creates a regular expression pattern used to tokenize text into individual tokens.
 * @param contractions - Contractions used by the tokenizer pattern.
 * @param rules - Rules used to build the tokenizer pattern.
 *
 * @see {@linkcode DEFAULT_TOKENIZER_RULES}
 */
export declare function createTokenizerPattern(contractions?: string[], rules?: string[]): RegExp;