'use strict';

const modelContextSizeMap = /* @__PURE__ */ new Map([
  ["gpt-3.5-turbo-16k", 16384],
  ["gpt-3.5-turbo", 4096],
  ["gpt-4-1106-preview", 128e3],
  ["gpt-4-32k", 32768],
  ["gpt-4", 8192],
  ["text-davinci-003", 4097],
  ["text-curie-001", 2048],
  ["text-babbage-001", 2048],
  ["text-ada-001", 2048],
  ["code-davinci-002", 8e3],
  ["code-cushman-001", 2048]
]);
function resolveModelName(modelName) {
  if (modelName.startsWith("gpt-3.5-turbo-16k"))
    return "gpt-3.5-turbo-16k";
  if (modelName.startsWith("gpt-3.5-turbo-"))
    return "gpt-3.5-turbo";
  if (modelName.startsWith("gpt-4-32k"))
    return "gpt-4-32k";
  if (modelName.startsWith("gpt-4-"))
    return "gpt-4";
  return modelName;
}
function getModelContextSize(modelName) {
  const modelKey = resolveModelName(modelName);
  return modelContextSizeMap.get(modelKey) ?? 4097;
}
function getEmbeddingContextSize(modelName) {
  if (modelName === "text-embedding-ada-002")
    return 8191;
  return 2046;
}
const WHITESPACE_RE = /^\s+$/;
const CJK_RE = /[\u4E00-\u9FFF\u3400-\u4DBF\u3000-\u303F\uFF00-\uFFEF\u30A0-\u30FF\u2E80-\u2EFF\u31C0-\u31EF\u3200-\u32FF\u3300-\u33FF\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F\uA960-\uA97F\uD7B0-\uD7FF]/;
const NUMERIC_SEQUENCE_RE = /[\d.,]+/;
const PUNCTUATION_RE = /[.,!?;'"„“”‘’\-(){}[\]<>:/\\|@#$%^&*+=`~]/;
const ALPHANUMERIC_RE = /^[a-zA-Z0-9\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]+$/;
const DEFAULT_AVERAGE_CHARS_PER_TOKEN = 6;
const LANGUAGE_METRICS = [
  { regex: /[äöüßẞ]/i, averageCharsPerToken: 3 }
];
function chunkByMaxTokens(input, maxTokens, overlap = 0) {
  const counts = approximateTokenChunks(input);
  const chunks = [];
  let chunk = [];
  let count = 0;
  for (const token of counts) {
    if (count + token.count > maxTokens) {
      chunks.push(chunk);
      if (overlap > 0) {
        let i = chunk.length - 1;
        count = 0;
        while (i-- && count < overlap) {
          count += chunk[i].count;
        }
        chunk = chunk.slice(i);
      } else {
        chunk = [];
        count = 0;
      }
    }
    chunk.push(token);
    count += token.count;
  }
  if (chunk.length > 0) {
    chunks.push(chunk);
  }
  return chunks.map((chunk2) => chunk2.map(({ token }) => token).join(""));
}
function approximateTokenChunks(input) {
  const roughTokens = input.split(/(\s+|[.,!?;'"„“”‘’\-(){}[\]<>:/\\|@#$%^&*+=`~]+)/).filter(Boolean);
  const tokenCounts = [];
  for (const token of roughTokens) {
    let averageCharsPerToken;
    for (const language of LANGUAGE_METRICS) {
      if (language.regex.test(token)) {
        averageCharsPerToken = language.averageCharsPerToken;
        break;
      }
    }
    let count = 0;
    if (WHITESPACE_RE.test(token)) {
      count = 0;
    } else if (CJK_RE.test(token)) {
      count = Array.from(token).length;
    } else if (NUMERIC_SEQUENCE_RE.test(token)) {
      count = 1;
    } else if (token.length <= 3) {
      count = 1;
    } else if (PUNCTUATION_RE.test(token)) {
      count = token.length > 1 ? Math.ceil(token.length / 2) : 1;
    } else if (ALPHANUMERIC_RE.test(token) || averageCharsPerToken) {
      count = Math.ceil(token.length / (averageCharsPerToken ?? DEFAULT_AVERAGE_CHARS_PER_TOKEN));
    } else {
      count = Array.from(token).length;
    }
    tokenCounts.push({ token, count });
  }
  return tokenCounts;
}
function approximateTokenSize(input) {
  return approximateTokenChunks(input).reduce((acc, { count }) => acc + count, 0);
}
function approximateMaxTokenSize({
  prompt,
  modelName,
  maxTokensInResponse = 0
}) {
  const remainingTokens = getModelContextSize(modelName) - approximateTokenSize(prompt) - maxTokensInResponse;
  return Math.max(0, remainingTokens);
}
function isWithinTokenLimit(input, tokenLimit) {
  return approximateTokenSize(input) <= tokenLimit;
}

exports.approximateMaxTokenSize = approximateMaxTokenSize;
exports.approximateTokenChunks = approximateTokenChunks;
exports.approximateTokenSize = approximateTokenSize;
exports.chunkByMaxTokens = chunkByMaxTokens;
exports.getEmbeddingContextSize = getEmbeddingContextSize;
exports.getModelContextSize = getModelContextSize;
exports.isWithinTokenLimit = isWithinTokenLimit;
exports.resolveModelName = resolveModelName;