/**
 * @copyright Sister Software. All rights reserved.
 * @author Teffen Ellis, et al.
 * @license
 * See LICENSE file in the project root for full license information.
 */
import { TokenEncodingsRecord } from '../BytePairTokenMap.mjs';
import { BPEVocab } from '../RanksMap.mjs';
/**
 * Parses a BPE file into a list of bigrams
 *
 * The vocab.bpe file is a text file that contains a set of byte pair encoding (BPE) codes
 * that are used in the tokenization process.
 *
 * The file should be in the following format:
 *
 * ```text
 * #version: VERSION_STRING
 * [prefix1] [suffix1]
 * [prefixN] [suffixN]
 * ...
 * ```
 */
export declare function parseBPEFile(bpeFileContents: string): BPEVocab;
/**
 * Parse a token encoder file, usually from a file named `encoder.json`
 */
export declare function parseEncoderFile(
/**
 * The token encoder content, either as a string or as a parsed object.
 */
tokenEncoderContent: string | TokenEncodingsRecord): TokenEncodingsRecord;
export interface TikTokenEntry {
    token: string;
    rank: number;
}
export declare function parseTikTokenFile(tikTokenFileContents: string): TikTokenEntry[];