'use strict'; const modelContextSizeMap = /* @__PURE__ */ new Map([ ["gpt-3.5-turbo-16k", 16384], ["gpt-3.5-turbo", 4096], ["gpt-4-1106-preview", 128e3], ["gpt-4-32k", 32768], ["gpt-4", 8192], ["text-davinci-003", 4097], ["text-curie-001", 2048], ["text-babbage-001", 2048], ["text-ada-001", 2048], ["code-davinci-002", 8e3], ["code-cushman-001", 2048] ]); function resolveModelName(modelName) { if (modelName.startsWith("gpt-3.5-turbo-16k")) return "gpt-3.5-turbo-16k"; if (modelName.startsWith("gpt-3.5-turbo-")) return "gpt-3.5-turbo"; if (modelName.startsWith("gpt-4-32k")) return "gpt-4-32k"; if (modelName.startsWith("gpt-4-")) return "gpt-4"; return modelName; } function getModelContextSize(modelName) { const modelKey = resolveModelName(modelName); return modelContextSizeMap.get(modelKey) ?? 4097; } function getEmbeddingContextSize(modelName) { if (modelName === "text-embedding-ada-002") return 8191; return 2046; } const WHITESPACE_RE = /^\s+$/; const CJK_RE = /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/; const NUMERIC_SEQUENCE_RE = /[\d.,]+/; const PUNCTUATION_RE = /[.,!?;'"„“”‘’\-(){}[\]<>:/\\|@#$%^&*+=`~]/; const ALPHANUMERIC_RE = /^[a-zA-Z0-9\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]+$/; const DEFAULT_AVERAGE_CHARS_PER_TOKEN = 6; const LANGUAGE_METRICS = [ { regex: /[äöüßẞ]/i, averageCharsPerToken: 3 } ]; function approximateTokenSize(input) { const roughTokens = input.split(/(\s+|[.,!?;'"„“”‘’\-(){}[\]<>:/\\|@#$%^&*+=`~]+)/).filter(Boolean); let tokenCount = 0; for (const token of roughTokens) { let averageCharsPerToken; for (const language of LANGUAGE_METRICS) { if (language.regex.test(token)) { averageCharsPerToken = language.averageCharsPerToken; break; } } if (WHITESPACE_RE.test(token)) { continue; } else if (CJK_RE.test(token)) { tokenCount += Array.from(token).length; } else if (NUMERIC_SEQUENCE_RE.test(token)) { tokenCount += 1; } else if (token.length <= 3) { tokenCount += 1; } else if (PUNCTUATION_RE.test(token)) { tokenCount += token.length > 1 ? Math.ceil(token.length / 2) : 1; } else if (ALPHANUMERIC_RE.test(token) || averageCharsPerToken) { tokenCount += Math.ceil(token.length / (averageCharsPerToken ?? DEFAULT_AVERAGE_CHARS_PER_TOKEN)); } else { tokenCount += Array.from(token).length; } } return tokenCount; } function approximateMaxTokenSize({ prompt, modelName, maxTokensInResponse = 0 }) { const remainingTokens = getModelContextSize(modelName) - approximateTokenSize(prompt) - maxTokensInResponse; return Math.max(0, remainingTokens); } function isWithinTokenLimit(input, tokenLimit) { return approximateTokenSize(input) <= tokenLimit; } exports.approximateMaxTokenSize = approximateMaxTokenSize; exports.approximateTokenSize = approximateTokenSize; exports.getEmbeddingContextSize = getEmbeddingContextSize; exports.getModelContextSize = getModelContextSize; exports.isWithinTokenLimit = isWithinTokenLimit; exports.resolveModelName = resolveModelName;