{"version":3,"file":"friendli.cjs","names":["LLM","convertEventStreamToIterableReadableDataStream","GenerationChunk"],"sources":["../../src/llms/friendli.ts"],"sourcesContent":["import { CallbackManagerForLLMRun } from \"@langchain/core/callbacks/manager\";\nimport {\n type BaseLLMCallOptions,\n type BaseLLMParams,\n LLM,\n} from \"@langchain/core/language_models/llms\";\nimport { GenerationChunk } from \"@langchain/core/outputs\";\nimport { getEnvironmentVariable } from \"@langchain/core/utils/env\";\nimport { convertEventStreamToIterableReadableDataStream } from \"../utils/event_source_parse.js\";\n\n/**\n * The FriendliParams interface defines the input parameters for\n * the Friendli class.\n */\nexport interface FriendliParams extends BaseLLMParams {\n /**\n * Model name to use.\n */\n model?: string;\n /**\n * Base endpoint url.\n */\n baseUrl?: string;\n /**\n * Friendli personal access token to run as.\n */\n friendliToken?: string;\n /**\n * Friendli team ID to run as.\n */\n friendliTeam?: string;\n /**\n * Number between -2.0 and 2.0. Positive values penalizes tokens that have been\n * sampled, taking into account their frequency in the preceding text. This\n * penalization diminishes the model's tendency to reproduce identical lines\n * verbatim.\n */\n frequencyPenalty?: number;\n /**\n * Number between -2.0 and 2.0. Positive values penalizes tokens that have been\n * sampled at least once in the existing text.\n * presence_penalty: Optional[float] = None\n * The maximum number of tokens to generate. The length of your input tokens plus\n * `max_tokens` should not exceed the model's maximum length (e.g., 2048 for OpenAI\n * GPT-3)\n */\n maxTokens?: number;\n /**\n * When one of the stop phrases appears in the generation result, the API will stop\n * generation. The phrase is included in the generated result. If you are using\n * beam search, all of the active beams should contain the stop phrase to terminate\n * generation. Before checking whether a stop phrase is included in the result, the\n * phrase is converted into tokens.\n */\n stop?: string[];\n /**\n * Sampling temperature. Smaller temperature makes the generation result closer to\n * greedy, argmax (i.e., `top_k = 1`) sampling. If it is `None`, then 1.0 is used.\n */\n temperature?: number;\n /**\n * Tokens comprising the top `top_p` probability mass are kept for sampling. Numbers\n * between 0.0 (exclusive) and 1.0 (inclusive) are allowed. If it is `None`, then 1.0\n * is used by default.\n */\n topP?: number;\n /**\n * Additional kwargs to pass to the model.\n */\n modelKwargs?: Record;\n}\n\n/**\n * The Friendli class is used to interact with Friendli inference Endpoint models.\n * This requires your Friendli Token and Friendli Team which is autoloaded if not specified.\n */\nexport class Friendli extends LLM {\n lc_serializable = true;\n\n static lc_name() {\n return \"Friendli\";\n }\n\n get lc_secrets(): { [key: string]: string } | undefined {\n return {\n friendliToken: \"FRIENDLI_TOKEN\",\n friendliTeam: \"FRIENDLI_TEAM\",\n };\n }\n\n model = \"mixtral-8x7b-instruct-v0-1\";\n\n baseUrl = \"https://inference.friendli.ai\";\n\n friendliToken?: string;\n\n friendliTeam?: string;\n\n frequencyPenalty?: number;\n\n maxTokens?: number;\n\n stop?: string[];\n\n temperature?: number;\n\n topP?: number;\n\n modelKwargs?: Record;\n\n constructor(fields: FriendliParams) {\n super(fields);\n\n this.model = fields?.model ?? this.model;\n this.baseUrl = fields?.baseUrl ?? this.baseUrl;\n this.friendliToken =\n fields?.friendliToken ?? getEnvironmentVariable(\"FRIENDLI_TOKEN\");\n this.friendliTeam =\n fields?.friendliTeam ?? getEnvironmentVariable(\"FRIENDLI_TEAM\");\n this.frequencyPenalty = fields?.frequencyPenalty ?? this.frequencyPenalty;\n this.maxTokens = fields?.maxTokens ?? this.maxTokens;\n this.stop = fields?.stop ?? this.stop;\n this.temperature = fields?.temperature ?? this.temperature;\n this.topP = fields?.topP ?? this.topP;\n this.modelKwargs = fields?.modelKwargs ?? {};\n\n if (!this.friendliToken) {\n throw new Error(\"Missing Friendli Token\");\n }\n }\n\n _llmType() {\n return \"friendli\";\n }\n\n private constructHeaders(stream: boolean) {\n return {\n \"Content-Type\": \"application/json\",\n Accept: stream ? \"text/event-stream\" : \"application/json\",\n Authorization: `Bearer ${this.friendliToken}`,\n \"X-Friendli-Team\": this.friendliTeam ?? \"\",\n };\n }\n\n private constructBody(\n prompt: string,\n stream: boolean,\n _options?: this[\"ParsedCallOptions\"]\n ) {\n const body = JSON.stringify({\n prompt,\n stream,\n model: this.model,\n max_tokens: this.maxTokens,\n frequency_penalty: this.frequencyPenalty,\n stop: this.stop,\n temperature: this.temperature,\n top_p: this.topP,\n ...this.modelKwargs,\n });\n return body;\n }\n\n /**\n * Calls the Friendli endpoint and retrieves the result.\n * @param {string} prompt The input prompt.\n * @returns {Promise} A promise that resolves to the generated string.\n */\n /** @ignore */\n async _call(\n prompt: string,\n _options: this[\"ParsedCallOptions\"]\n ): Promise {\n interface FriendliResponse {\n choices: {\n index: number;\n seed: number;\n text: string;\n tokens: number[];\n }[];\n usage: {\n prompt_tokens: number;\n completion_tokens: number;\n total_tokens: number;\n };\n }\n\n const response = (await this.caller.call(async () =>\n fetch(`${this.baseUrl}/v1/completions`, {\n method: \"POST\",\n headers: this.constructHeaders(false),\n body: this.constructBody(prompt, false, _options),\n }).then((res) => res.json())\n )) as FriendliResponse;\n\n return response.choices[0].text;\n }\n\n async *_streamResponseChunks(\n prompt: string,\n _options: this[\"ParsedCallOptions\"],\n runManager?: CallbackManagerForLLMRun\n ): AsyncGenerator {\n interface FriendliResponse {\n event: string;\n index: number;\n text: string;\n token: number;\n }\n\n interface FriendliCompleteResponse {\n event: string;\n choices: {\n index: number;\n seed: number;\n text: string;\n tokens: number[];\n }[];\n usage: {\n prompt_tokens: number;\n completion_tokens: number;\n total_tokens: number;\n };\n }\n\n const response = await this.caller.call(async () =>\n fetch(`${this.baseUrl}/v1/completions`, {\n method: \"POST\",\n headers: this.constructHeaders(true),\n body: this.constructBody(prompt, true, _options),\n })\n );\n\n if (response.status !== 200 || !response.body) {\n const errorResponse = await response.json();\n throw new Error(JSON.stringify(errorResponse));\n }\n\n const stream = convertEventStreamToIterableReadableDataStream(\n response.body\n );\n\n for await (const chunk of stream) {\n if (chunk.event !== \"complete\") {\n const parsedChunk = JSON.parse(chunk) as FriendliResponse;\n const generationChunk = new GenerationChunk({\n text: parsedChunk.text ?? \"\",\n });\n\n yield generationChunk;\n\n // eslint-disable-next-line no-void\n void runManager?.handleLLMNewToken(generationChunk.text ?? \"\");\n } else {\n const parsedChunk = JSON.parse(chunk) as FriendliCompleteResponse;\n const generationChunk = new GenerationChunk({\n text: \"\",\n generationInfo: {\n choices: parsedChunk.choices,\n usage: parsedChunk.usage,\n },\n });\n\n yield generationChunk;\n }\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;AA4EA,IAAa,WAAb,cAA8BA,qCAAAA,IAAwB;CACpD,kBAAkB;CAElB,OAAO,UAAU;AACf,SAAO;;CAGT,IAAI,aAAoD;AACtD,SAAO;GACL,eAAe;GACf,cAAc;GACf;;CAGH,QAAQ;CAER,UAAU;CAEV;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,YAAY,QAAwB;AAClC,QAAM,OAAO;AAEb,OAAK,QAAQ,QAAQ,SAAS,KAAK;AACnC,OAAK,UAAU,QAAQ,WAAW,KAAK;AACvC,OAAK,gBACH,QAAQ,kBAAA,GAAA,0BAAA,wBAAwC,iBAAiB;AACnE,OAAK,eACH,QAAQ,iBAAA,GAAA,0BAAA,wBAAuC,gBAAgB;AACjE,OAAK,mBAAmB,QAAQ,oBAAoB,KAAK;AACzD,OAAK,YAAY,QAAQ,aAAa,KAAK;AAC3C,OAAK,OAAO,QAAQ,QAAQ,KAAK;AACjC,OAAK,cAAc,QAAQ,eAAe,KAAK;AAC/C,OAAK,OAAO,QAAQ,QAAQ,KAAK;AACjC,OAAK,cAAc,QAAQ,eAAe,EAAE;AAE5C,MAAI,CAAC,KAAK,cACR,OAAM,IAAI,MAAM,yBAAyB;;CAI7C,WAAW;AACT,SAAO;;CAGT,iBAAyB,QAAiB;AACxC,SAAO;GACL,gBAAgB;GAChB,QAAQ,SAAS,sBAAsB;GACvC,eAAe,UAAU,KAAK;GAC9B,mBAAmB,KAAK,gBAAgB;GACzC;;CAGH,cACE,QACA,QACA,UACA;AAYA,SAXa,KAAK,UAAU;GAC1B;GACA;GACA,OAAO,KAAK;GACZ,YAAY,KAAK;GACjB,mBAAmB,KAAK;GACxB,MAAM,KAAK;GACX,aAAa,KAAK;GAClB,OAAO,KAAK;GACZ,GAAG,KAAK;GACT,CAAC;;;;;;;;CAUJ,MAAM,MACJ,QACA,UACiB;AAuBjB,UARkB,MAAM,KAAK,OAAO,KAAK,YACvC,MAAM,GAAG,KAAK,QAAQ,kBAAkB;GACtC,QAAQ;GACR,SAAS,KAAK,iBAAiB,MAAM;GACrC,MAAM,KAAK,cAAc,QAAQ,OAAO,SAAS;GAClD,CAAC,CAAC,MAAM,QAAQ,IAAI,MAAM,CAAC,CAC7B,EAEe,QAAQ,GAAG;;CAG7B,OAAO,sBACL,QACA,UACA,YACiC;EAuBjC,MAAM,WAAW,MAAM,KAAK,OAAO,KAAK,YACtC,MAAM,GAAG,KAAK,QAAQ,kBAAkB;GACtC,QAAQ;GACR,SAAS,KAAK,iBAAiB,KAAK;GACpC,MAAM,KAAK,cAAc,QAAQ,MAAM,SAAS;GACjD,CAAC,CACH;AAED,MAAI,SAAS,WAAW,OAAO,CAAC,SAAS,MAAM;GAC7C,MAAM,gBAAgB,MAAM,SAAS,MAAM;AAC3C,SAAM,IAAI,MAAM,KAAK,UAAU,cAAc,CAAC;;EAGhD,MAAM,SAASC,iCAAAA,+CACb,SAAS,KACV;AAED,aAAW,MAAM,SAAS,OACxB,KAAI,MAAM,UAAU,YAAY;GAE9B,MAAM,kBAAkB,IAAIC,wBAAAA,gBAAgB,EAC1C,MAFkB,KAAK,MAAM,MAAM,CAEjB,QAAQ,IAC3B,CAAC;AAEF,SAAM;AAGD,eAAY,kBAAkB,gBAAgB,QAAQ,GAAG;SACzD;GACL,MAAM,cAAc,KAAK,MAAM,MAAM;AASrC,SARwB,IAAIA,wBAAAA,gBAAgB;IAC1C,MAAM;IACN,gBAAgB;KACd,SAAS,YAAY;KACrB,OAAO,YAAY;KACpB;IACF,CAAC"}