{"version":3,"file":"llama_cpp.cjs","names":["LLM","inputs: LlamaCppInputs","createLlamaModel","createLlamaContext","createLlamaJsonSchemaGrammar","createCustomGrammar","createLlamaSession","prompt: string","options?: this[\"ParsedCallOptions\"]","_options: this[\"ParsedCallOptions\"]","runManager?: CallbackManagerForLLMRun","GenerationChunk"],"sources":["../../src/llms/llama_cpp.ts"],"sourcesContent":["import {\n  LlamaModel,\n  LlamaContext,\n  LlamaChatSession,\n  LlamaJsonSchemaGrammar,\n  LlamaGrammar,\n  getLlama,\n  GbnfJsonSchema,\n} from \"node-llama-cpp\";\nimport {\n  LLM,\n  type BaseLLMCallOptions,\n  type BaseLLMParams,\n} from \"@langchain/core/language_models/llms\";\nimport { CallbackManagerForLLMRun } from \"@langchain/core/callbacks/manager\";\nimport { GenerationChunk } from \"@langchain/core/outputs\";\n\nimport {\n  LlamaBaseCppInputs,\n  createLlamaModel,\n  createLlamaContext,\n  createLlamaSession,\n  createLlamaJsonSchemaGrammar,\n  createCustomGrammar,\n} from \"../utils/llama_cpp.js\";\n\n/**\n * Note that the modelPath is the only required parameter. For testing you\n * can set this in the environment variable `LLAMA_PATH`.\n */\nexport interface LlamaCppInputs extends LlamaBaseCppInputs, BaseLLMParams {}\n\nexport interface LlamaCppCallOptions extends BaseLLMCallOptions {\n  /** The maximum number of tokens the response should contain. */\n  maxTokens?: number;\n  /** A function called when matching the provided token array */\n  onToken?: (tokens: number[]) => void;\n}\n\n/**\n *  To use this model you need to have the `node-llama-cpp` module installed.\n *  This can be installed using `npm install -S node-llama-cpp` and the minimum\n *  version supported in version 2.0.0.\n *  This also requires that have a locally built version of Llama3 installed.\n */\nexport class LlamaCpp extends LLM<LlamaCppCallOptions> {\n  lc_serializable = true;\n\n  static inputs: LlamaCppInputs;\n\n  maxTokens?: number;\n\n  temperature?: number;\n\n  topK?: number;\n\n  topP?: number;\n\n  trimWhitespaceSuffix?: boolean;\n\n  _model: LlamaModel;\n\n  _context: LlamaContext;\n\n  _session: LlamaChatSession;\n\n  _jsonSchema: LlamaJsonSchemaGrammar<GbnfJsonSchema> | undefined;\n\n  _gbnf: LlamaGrammar | undefined;\n\n  static lc_name() {\n    return \"LlamaCpp\";\n  }\n\n  public constructor(inputs: LlamaCppInputs) {\n    super(inputs);\n    this.maxTokens = inputs?.maxTokens;\n    this.temperature = inputs?.temperature;\n    this.topK = inputs?.topK;\n    this.topP = inputs?.topP;\n    this.trimWhitespaceSuffix = inputs?.trimWhitespaceSuffix;\n  }\n\n  /**\n   * Initializes the llama_cpp model for usage.\n   * @param inputs - the inputs passed onto the model.\n   * @returns A Promise that resolves to the LlamaCpp type class.\n   */\n  public static async initialize(inputs: LlamaCppInputs): Promise<LlamaCpp> {\n    const instance = new LlamaCpp(inputs);\n    const llama = await getLlama();\n\n    instance._model = await createLlamaModel(inputs, llama);\n    instance._context = await createLlamaContext(instance._model, inputs);\n    instance._jsonSchema = await createLlamaJsonSchemaGrammar(\n      inputs?.jsonSchema,\n      llama\n    );\n    instance._gbnf = await createCustomGrammar(inputs?.gbnf, llama);\n    instance._session = createLlamaSession(instance._context);\n\n    return instance;\n  }\n\n  _llmType() {\n    return \"llama_cpp\";\n  }\n\n  /** @ignore */\n  async _call(\n    prompt: string,\n    options?: this[\"ParsedCallOptions\"]\n  ): Promise<string> {\n    try {\n      let promptGrammer;\n\n      if (this._jsonSchema !== undefined) {\n        promptGrammer = this._jsonSchema;\n      } else if (this._gbnf !== undefined) {\n        promptGrammer = this._gbnf;\n      } else {\n        promptGrammer = undefined;\n      }\n      const promptOptions = {\n        grammar: promptGrammer,\n        onToken: options?.onToken,\n        maxTokens: this?.maxTokens,\n        temperature: this?.temperature,\n        topK: this?.topK,\n        topP: this?.topP,\n        trimWhitespaceSuffix: this?.trimWhitespaceSuffix,\n      };\n\n      const completion = await this._session.prompt(prompt, promptOptions);\n\n      if (this._jsonSchema !== undefined && completion !== undefined) {\n        return this._jsonSchema.parse(completion) as unknown as string;\n      }\n\n      return completion;\n    } catch {\n      throw new Error(\"Error getting prompt completion.\");\n    }\n  }\n\n  async *_streamResponseChunks(\n    prompt: string,\n    _options: this[\"ParsedCallOptions\"],\n    runManager?: CallbackManagerForLLMRun\n  ): AsyncGenerator<GenerationChunk> {\n    const promptOptions = {\n      temperature: this?.temperature,\n      maxTokens: this?.maxTokens,\n      topK: this?.topK,\n      topP: this?.topP,\n    };\n\n    if (this._context.sequencesLeft === 0) {\n      this._context = await createLlamaContext(this._model, LlamaCpp.inputs);\n    }\n    const sequence = this._context.getSequence();\n    const tokens = this._model.tokenize(prompt);\n\n    const stream = await this.caller.call(async () =>\n      sequence.evaluate(tokens, promptOptions)\n    );\n\n    for await (const chunk of stream) {\n      yield new GenerationChunk({\n        text: this._model.detokenize([chunk]),\n        generationInfo: {},\n      });\n      await runManager?.handleLLMNewToken(\n        this._model.detokenize([chunk]) ?? \"\"\n      );\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AA6CA,IAAa,WAAb,MAAa,iBAAiBA,0CAAyB;CACrD,kBAAkB;CAElB,OAAO;CAEP;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,OAAO,UAAU;AACf,SAAO;CACR;CAED,AAAO,YAAYC,QAAwB;EACzC,MAAM,OAAO;EACb,KAAK,YAAY,QAAQ;EACzB,KAAK,cAAc,QAAQ;EAC3B,KAAK,OAAO,QAAQ;EACpB,KAAK,OAAO,QAAQ;EACpB,KAAK,uBAAuB,QAAQ;CACrC;;;;;;CAOD,aAAoB,WAAWA,QAA2C;EACxE,MAAM,WAAW,IAAI,SAAS;EAC9B,MAAM,QAAQ,oCAAgB;EAE9B,SAAS,SAAS,MAAMC,mCAAiB,QAAQ,MAAM;EACvD,SAAS,WAAW,MAAMC,qCAAmB,SAAS,QAAQ,OAAO;EACrE,SAAS,cAAc,MAAMC,+CAC3B,QAAQ,YACR,MACD;EACD,SAAS,QAAQ,MAAMC,sCAAoB,QAAQ,MAAM,MAAM;EAC/D,SAAS,WAAWC,qCAAmB,SAAS,SAAS;AAEzD,SAAO;CACR;CAED,WAAW;AACT,SAAO;CACR;;CAGD,MAAM,MACJC,QACAC,SACiB;AACjB,MAAI;GACF,IAAI;AAEJ,OAAI,KAAK,gBAAgB,QACvB,gBAAgB,KAAK;YACZ,KAAK,UAAU,QACxB,gBAAgB,KAAK;QAErB,gBAAgB;GAElB,MAAM,gBAAgB;IACpB,SAAS;IACT,SAAS,SAAS;IAClB,WAAW,MAAM;IACjB,aAAa,MAAM;IACnB,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,sBAAsB,MAAM;GAC7B;GAED,MAAM,aAAa,MAAM,KAAK,SAAS,OAAO,QAAQ,cAAc;AAEpE,OAAI,KAAK,gBAAgB,UAAa,eAAe,OACnD,QAAO,KAAK,YAAY,MAAM,WAAW;AAG3C,UAAO;EACR,QAAO;AACN,SAAM,IAAI,MAAM;EACjB;CACF;CAED,OAAO,sBACLD,QACAE,UACAC,YACiC;EACjC,MAAM,gBAAgB;GACpB,aAAa,MAAM;GACnB,WAAW,MAAM;GACjB,MAAM,MAAM;GACZ,MAAM,MAAM;EACb;AAED,MAAI,KAAK,SAAS,kBAAkB,GAClC,KAAK,WAAW,MAAMP,qCAAmB,KAAK,QAAQ,SAAS,OAAO;EAExE,MAAM,WAAW,KAAK,SAAS,aAAa;EAC5C,MAAM,SAAS,KAAK,OAAO,SAAS,OAAO;EAE3C,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,YACpC,SAAS,SAAS,QAAQ,cAAc,CACzC;AAED,aAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,IAAIQ,yCAAgB;IACxB,MAAM,KAAK,OAAO,WAAW,CAAC,KAAM,EAAC;IACrC,gBAAgB,CAAE;GACnB;GACD,MAAM,YAAY,kBAChB,KAAK,OAAO,WAAW,CAAC,KAAM,EAAC,IAAI,GACpC;EACF;CACF;AACF"}