{"version":3,"file":"llama_cpp.cjs","names":["LLM","createLlamaModel","createLlamaContext","createLlamaJsonSchemaGrammar","createCustomGrammar","createLlamaSession","GenerationChunk"],"sources":["../../src/llms/llama_cpp.ts"],"sourcesContent":["import {\n  LlamaModel,\n  LlamaContext,\n  LlamaChatSession,\n  LlamaJsonSchemaGrammar,\n  LlamaGrammar,\n  getLlama,\n  GbnfJsonSchema,\n} from \"node-llama-cpp\";\nimport {\n  LLM,\n  type BaseLLMCallOptions,\n  type BaseLLMParams,\n} from \"@langchain/core/language_models/llms\";\nimport { CallbackManagerForLLMRun } from \"@langchain/core/callbacks/manager\";\nimport { GenerationChunk } from \"@langchain/core/outputs\";\n\nimport {\n  LlamaBaseCppInputs,\n  createLlamaModel,\n  createLlamaContext,\n  createLlamaSession,\n  createLlamaJsonSchemaGrammar,\n  createCustomGrammar,\n} from \"../utils/llama_cpp.js\";\n\n/**\n * Note that the modelPath is the only required parameter. For testing you\n * can set this in the environment variable `LLAMA_PATH`.\n */\nexport interface LlamaCppInputs extends LlamaBaseCppInputs, BaseLLMParams {}\n\nexport interface LlamaCppCallOptions extends BaseLLMCallOptions {\n  /** The maximum number of tokens the response should contain. */\n  maxTokens?: number;\n  /** A function called when matching the provided token array */\n  onToken?: (tokens: number[]) => void;\n}\n\n/**\n *  To use this model you need to have the `node-llama-cpp` module installed.\n *  This can be installed using `npm install -S node-llama-cpp` and the minimum\n *  version supported in version 2.0.0.\n *  This also requires that have a locally built version of Llama3 installed.\n */\nexport class LlamaCpp extends LLM<LlamaCppCallOptions> {\n  lc_serializable = true;\n\n  static inputs: LlamaCppInputs;\n\n  maxTokens?: number;\n\n  temperature?: number;\n\n  topK?: number;\n\n  topP?: number;\n\n  trimWhitespaceSuffix?: boolean;\n\n  _model: LlamaModel;\n\n  _context: LlamaContext;\n\n  _session: LlamaChatSession;\n\n  _jsonSchema: LlamaJsonSchemaGrammar<GbnfJsonSchema> | undefined;\n\n  _gbnf: LlamaGrammar | undefined;\n\n  static lc_name() {\n    return \"LlamaCpp\";\n  }\n\n  public constructor(inputs: LlamaCppInputs) {\n    super(inputs);\n    this.maxTokens = inputs?.maxTokens;\n    this.temperature = inputs?.temperature;\n    this.topK = inputs?.topK;\n    this.topP = inputs?.topP;\n    this.trimWhitespaceSuffix = inputs?.trimWhitespaceSuffix;\n  }\n\n  /**\n   * Initializes the llama_cpp model for usage.\n   * @param inputs - the inputs passed onto the model.\n   * @returns A Promise that resolves to the LlamaCpp type class.\n   */\n  public static async initialize(inputs: LlamaCppInputs): Promise<LlamaCpp> {\n    const instance = new LlamaCpp(inputs);\n    const llama = await getLlama();\n\n    instance._model = await createLlamaModel(inputs, llama);\n    instance._context = await createLlamaContext(instance._model, inputs);\n    instance._jsonSchema = await createLlamaJsonSchemaGrammar(\n      inputs?.jsonSchema,\n      llama\n    );\n    instance._gbnf = await createCustomGrammar(inputs?.gbnf, llama);\n    instance._session = createLlamaSession(instance._context);\n\n    return instance;\n  }\n\n  _llmType() {\n    return \"llama_cpp\";\n  }\n\n  /** @ignore */\n  async _call(\n    prompt: string,\n    options?: this[\"ParsedCallOptions\"]\n  ): Promise<string> {\n    try {\n      let promptGrammer;\n\n      if (this._jsonSchema !== undefined) {\n        promptGrammer = this._jsonSchema;\n      } else if (this._gbnf !== undefined) {\n        promptGrammer = this._gbnf;\n      } else {\n        promptGrammer = undefined;\n      }\n      const promptOptions = {\n        grammar: promptGrammer,\n        onToken: options?.onToken,\n        maxTokens: this?.maxTokens,\n        temperature: this?.temperature,\n        topK: this?.topK,\n        topP: this?.topP,\n        trimWhitespaceSuffix: this?.trimWhitespaceSuffix,\n      };\n\n      const completion = await this._session.prompt(prompt, promptOptions);\n\n      if (this._jsonSchema !== undefined && completion !== undefined) {\n        return this._jsonSchema.parse(completion) as unknown as string;\n      }\n\n      return completion;\n    } catch {\n      throw new Error(\"Error getting prompt completion.\");\n    }\n  }\n\n  async *_streamResponseChunks(\n    prompt: string,\n    _options: this[\"ParsedCallOptions\"],\n    runManager?: CallbackManagerForLLMRun\n  ): AsyncGenerator<GenerationChunk> {\n    const promptOptions = {\n      temperature: this?.temperature,\n      maxTokens: this?.maxTokens,\n      topK: this?.topK,\n      topP: this?.topP,\n    };\n\n    if (this._context.sequencesLeft === 0) {\n      this._context = await createLlamaContext(this._model, LlamaCpp.inputs);\n    }\n    const sequence = this._context.getSequence();\n    const tokens = this._model.tokenize(prompt);\n\n    const stream = await this.caller.call(async () =>\n      sequence.evaluate(tokens, promptOptions)\n    );\n\n    for await (const chunk of stream) {\n      yield new GenerationChunk({\n        text: this._model.detokenize([chunk]),\n        generationInfo: {},\n      });\n      await runManager?.handleLLMNewToken(\n        this._model.detokenize([chunk]) ?? \"\"\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AA6CA,IAAa,WAAb,MAAa,iBAAiBA,qCAAAA,IAAyB;CACrD,kBAAkB;CAElB,OAAO;CAEP;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,OAAO,UAAU;AACf,SAAO;;CAGT,YAAmB,QAAwB;AACzC,QAAM,OAAO;AACb,OAAK,YAAY,QAAQ;AACzB,OAAK,cAAc,QAAQ;AAC3B,OAAK,OAAO,QAAQ;AACpB,OAAK,OAAO,QAAQ;AACpB,OAAK,uBAAuB,QAAQ;;;;;;;CAQtC,aAAoB,WAAW,QAA2C;EACxE,MAAM,WAAW,IAAI,SAAS,OAAO;EACrC,MAAM,QAAQ,OAAA,GAAA,eAAA,WAAgB;AAE9B,WAAS,SAAS,MAAMC,kBAAAA,iBAAiB,QAAQ,MAAM;AACvD,WAAS,WAAW,MAAMC,kBAAAA,mBAAmB,SAAS,QAAQ,OAAO;AACrE,WAAS,cAAc,MAAMC,kBAAAA,6BAC3B,QAAQ,YACR,MACD;AACD,WAAS,QAAQ,MAAMC,kBAAAA,oBAAoB,QAAQ,MAAM,MAAM;AAC/D,WAAS,WAAWC,kBAAAA,mBAAmB,SAAS,SAAS;AAEzD,SAAO;;CAGT,WAAW;AACT,SAAO;;;CAIT,MAAM,MACJ,QACA,SACiB;AACjB,MAAI;GACF,IAAI;AAEJ,OAAI,KAAK,gBAAgB,KAAA,EACvB,iBAAgB,KAAK;YACZ,KAAK,UAAU,KAAA,EACxB,iBAAgB,KAAK;OAErB,iBAAgB,KAAA;GAElB,MAAM,gBAAgB;IACpB,SAAS;IACT,SAAS,SAAS;IAClB,WAAW,MAAM;IACjB,aAAa,MAAM;IACnB,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,sBAAsB,MAAM;IAC7B;GAED,MAAM,aAAa,MAAM,KAAK,SAAS,OAAO,QAAQ,cAAc;AAEpE,OAAI,KAAK,gBAAgB,KAAA,KAAa,eAAe,KAAA,EACnD,QAAO,KAAK,YAAY,MAAM,WAAW;AAG3C,UAAO;UACD;AACN,SAAM,IAAI,MAAM,mCAAmC;;;CAIvD,OAAO,sBACL,QACA,UACA,YACiC;EACjC,MAAM,gBAAgB;GACpB,aAAa,MAAM;GACnB,WAAW,MAAM;GACjB,MAAM,MAAM;GACZ,MAAM,MAAM;GACb;AAED,MAAI,KAAK,SAAS,kBAAkB,EAClC,MAAK,WAAW,MAAMH,kBAAAA,mBAAmB,KAAK,QAAQ,SAAS,OAAO;EAExE,MAAM,WAAW,KAAK,SAAS,aAAa;EAC5C,MAAM,SAAS,KAAK,OAAO,SAAS,OAAO;EAE3C,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,YACpC,SAAS,SAAS,QAAQ,cAAc,CACzC;AAED,aAAW,MAAM,SAAS,QAAQ;AAChC,SAAM,IAAII,wBAAAA,gBAAgB;IACxB,MAAM,KAAK,OAAO,WAAW,CAAC,MAAM,CAAC;IACrC,gBAAgB,EAAE;IACnB,CAAC;AACF,SAAM,YAAY,kBAChB,KAAK,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,GACpC"}