{"version":3,"file":"Model.mjs","sources":["webpack://@ui-tars/sdk/./src/Model.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport OpenAI, { type ClientOptions } from 'openai';\nimport {\n type ChatCompletionCreateParamsNonStreaming,\n type ChatCompletionCreateParamsBase,\n type ChatCompletionMessageParam,\n} from 'openai/resources/chat/completions';\nimport { actionParser } from '@ui-tars/action-parser';\n\nimport { useContext } from './context/useContext';\nimport { Model, type InvokeParams, type InvokeOutput } from './types';\n\nimport { preprocessResizeImage, convertToOpenAIMessages } from './utils';\nimport { DEFAULT_FACTORS } from './constants';\nimport {\n UITarsModelVersion,\n MAX_PIXELS_V1_0,\n MAX_PIXELS_V1_5,\n MAX_PIXELS_DOUBAO,\n} from '@ui-tars/shared/types';\n\ntype OpenAIChatCompletionCreateParams = Omit &\n Pick<\n ChatCompletionCreateParamsBase,\n 'model' | 'max_tokens' | 'temperature' | 'top_p'\n >;\n\nexport interface UITarsModelConfig extends OpenAIChatCompletionCreateParams {}\n\nexport interface ThinkingVisionProModelConfig\n extends ChatCompletionCreateParamsNonStreaming {\n thinking?: {\n type: 'enabled' | 'disabled';\n };\n}\n\nexport class UITarsModel extends Model {\n constructor(protected readonly modelConfig: UITarsModelConfig) {\n super();\n this.modelConfig = modelConfig;\n }\n\n /** [widthFactor, heightFactor] */\n get factors(): [number, number] {\n return DEFAULT_FACTORS;\n }\n\n get modelName(): string {\n return this.modelConfig.model ?? 'unknown';\n }\n\n /**\n * call real LLM / VLM Model\n * @param params\n * @param options\n * @returns\n */\n protected async invokeModelProvider(\n uiTarsVersion: UITarsModelVersion = UITarsModelVersion.V1_0,\n params: {\n messages: Array;\n },\n options: {\n signal?: AbortSignal;\n },\n ): Promise<{\n prediction: string;\n costTime?: number;\n costTokens?: number;\n }> {\n const { messages } = params;\n const {\n baseURL,\n apiKey,\n model,\n max_tokens = uiTarsVersion == UITarsModelVersion.V1_5 ? 65535 : 1000,\n temperature = 0,\n top_p = 0.7,\n ...restOptions\n } = this.modelConfig;\n\n const openai = new OpenAI({\n ...restOptions,\n maxRetries: 0,\n baseURL,\n apiKey,\n });\n\n const createCompletionPrams: ChatCompletionCreateParamsNonStreaming = {\n model,\n messages,\n stream: false,\n seed: null,\n stop: null,\n frequency_penalty: null,\n presence_penalty: null,\n // custom options\n max_tokens,\n temperature,\n top_p,\n };\n\n const createCompletionPramsThinkingVp: ThinkingVisionProModelConfig = {\n ...createCompletionPrams,\n thinking: {\n type: 'disabled',\n },\n };\n\n const startTime = Date.now();\n const result = await openai.chat.completions.create(\n createCompletionPramsThinkingVp,\n options,\n );\n const costTime = Date.now() - startTime;\n\n return {\n prediction: result.choices?.[0]?.message?.content ?? '',\n costTime: costTime,\n costTokens: result.usage?.total_tokens ?? 0,\n };\n }\n\n async invoke(params: InvokeParams): Promise {\n const { conversations, images, screenContext, scaleFactor, uiTarsVersion } =\n params;\n const { logger, signal } = useContext();\n\n logger?.info(\n `[UITarsModel] invoke: screenContext=${JSON.stringify(screenContext)}, scaleFactor=${scaleFactor}, uiTarsVersion=${uiTarsVersion}`,\n );\n\n const maxPixels =\n uiTarsVersion === UITarsModelVersion.V1_5\n ? MAX_PIXELS_V1_5\n : uiTarsVersion === UITarsModelVersion.DOUBAO_1_5_15B ||\n uiTarsVersion === UITarsModelVersion.DOUBAO_1_5_20B\n ? MAX_PIXELS_DOUBAO\n : MAX_PIXELS_V1_0;\n const compressedImages = await Promise.all(\n images.map((image) => preprocessResizeImage(image, maxPixels)),\n );\n\n const messages = convertToOpenAIMessages({\n conversations,\n images: compressedImages,\n });\n\n const startTime = Date.now();\n const result = await this.invokeModelProvider(\n uiTarsVersion,\n {\n messages,\n },\n {\n signal,\n },\n )\n .catch((e) => {\n logger?.error('[UITarsModel] error', e);\n throw e;\n })\n .finally(() => {\n logger?.info(`[UITarsModel cost]: ${Date.now() - startTime}ms`);\n });\n\n if (!result.prediction) {\n const err = new Error();\n err.name = 'vlm response error';\n err.stack = JSON.stringify(result) ?? 'no message';\n logger?.error(err);\n throw err;\n }\n\n const { prediction, costTime, costTokens } = result;\n\n try {\n const { parsed: parsedPredictions } = await actionParser({\n prediction,\n factor: this.factors,\n screenContext,\n scaleFactor,\n modelVer: uiTarsVersion,\n });\n return {\n prediction,\n parsedPredictions,\n costTime,\n costTokens,\n };\n } catch (error) {\n logger?.error('[UITarsModel] error', error);\n return {\n prediction,\n parsedPredictions: [],\n };\n }\n }\n}\n"],"names":["UITarsModel","Model","DEFAULT_FACTORS","uiTarsVersion","UITarsModelVersion","params","options","_result_choices__message","_result_usage","messages","baseURL","apiKey","model","max_tokens","temperature","top_p","restOptions","openai","OpenAI","createCompletionPrams","createCompletionPramsThinkingVp","startTime","Date","result","costTime","conversations","images","screenContext","scaleFactor","logger","signal","useContext","JSON","maxPixels","MAX_PIXELS_V1_5","MAX_PIXELS_DOUBAO","MAX_PIXELS_V1_0","compressedImages","Promise","image","preprocessResizeImage","convertToOpenAIMessages","e","err","Error","prediction","costTokens","parsedPredictions","actionParser","error","modelConfig"],"mappings":";;;;;;;;;;;AAGC;;;;;;;;;;AAoCM,MAAMA,oBAAoBC,gDAAAA,KAAKA;IAOpC,IAAI,UAA4B;QAC9B,OAAOC,oDAAAA,eAAeA;IACxB;IAEA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,IAAI;IACnC;IAQA,MAAgB,oBACdC,gBAAoCC,2DAAAA,kBAAAA,CAAAA,IAAuB,EAC3DC,MAEC,EACDC,OAEC,EAKA;YAgDaC,0BAAAA,kBAAAA,iBAEAC;QAjDd,MAAM,EAAEC,QAAQ,EAAE,GAAGJ;QACrB,MAAM,EACJK,OAAO,EACPC,MAAM,EACNC,KAAK,EACLC,aAAaV,iBAAiBC,2DAAAA,kBAAAA,CAAAA,IAAuB,GAAG,QAAQ,IAAI,EACpEU,cAAc,CAAC,EACfC,QAAQ,GAAG,EACX,GAAGC,aACJ,GAAG,IAAI,CAAC,WAAW;QAEpB,MAAMC,SAAS,IAAIC,kCAAAA,CAAAA,UAAMA,CAAC;YACxB,GAAGF,WAAW;YACd,YAAY;YACZN;YACAC;QACF;QAEA,MAAMQ,wBAAgE;YACpEP;YACAH;YACA,QAAQ;YACR,MAAM;YACN,MAAM;YACN,mBAAmB;YACnB,kBAAkB;YAElBI;YACAC;YACAC;QACF;QAEA,MAAMK,kCAAgE;YACpE,GAAGD,qBAAqB;YACxB,UAAU;gBACR,MAAM;YACR;QACF;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAMC,SAAS,MAAMN,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,CACjDG,iCACAd;QAEF,MAAMkB,WAAWF,KAAK,GAAG,KAAKD;QAE9B,OAAO;YACL,YAAYd,AAAAA,CAAc,SAAdA,CAAAA,kBAAAA,OAAO,OAAO,AAAD,KAAbA,AAAAA,KAAAA,MAAAA,kBAAAA,KAAAA,IAAAA,AAAmB,SAAnBA,CAAAA,mBAAAA,eAAgB,CAAC,EAAE,AAAD,KAAlBA,AAAAA,KAAAA,MAAAA,mBAAAA,KAAAA,IAAAA,AAA4B,SAA5BA,CAAAA,2BAAAA,iBAAqB,OAAO,AAAD,KAA3BA,AAAAA,KAAAA,MAAAA,2BAAAA,KAAAA,IAAAA,yBAA8B,OAAO,AAAD,KAAK;YACrD,UAAUiB;YACV,YAAYhB,AAAAA,CAAY,SAAZA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,KAAXA,AAAAA,KAAAA,MAAAA,gBAAAA,KAAAA,IAAAA,cAAc,YAAY,AAAD,KAAK;QAC5C;IACF;IAEA,MAAM,OAAOH,MAAoB,EAAyB;QACxD,MAAM,EAAEoB,aAAa,EAAEC,MAAM,EAAEC,aAAa,EAAEC,WAAW,EAAEzB,aAAa,EAAE,GACxEE;QACF,MAAM,EAAEwB,MAAM,EAAEC,MAAM,EAAE,GAAGC,AAAAA,IAAAA,6DAAAA,UAAAA,AAAAA;QAE3BF,QAAAA,UAAAA,OAAQ,IAAI,CACV,CAAC,oCAAoC,EAAEG,KAAK,SAAS,CAACL,eAAe,cAAc,EAAEC,YAAY,gBAAgB,EAAEzB,eAAe;QAGpI,MAAM8B,YACJ9B,kBAAkBC,2DAAAA,kBAAAA,CAAAA,IAAuB,GACrC8B,2DAAAA,eAAeA,GACf/B,kBAAkBC,2DAAAA,kBAAAA,CAAAA,cAAiC,IACjDD,kBAAkBC,2DAAAA,kBAAAA,CAAAA,cAAiC,GACnD+B,2DAAAA,iBAAiBA,GACjBC,2DAAAA,eAAeA;QACvB,MAAMC,mBAAmB,MAAMC,QAAQ,GAAG,CACxCZ,OAAO,GAAG,CAAC,CAACa,QAAUC,AAAAA,IAAAA,gDAAAA,qBAAAA,AAAAA,EAAsBD,OAAON;QAGrD,MAAMxB,WAAWgC,AAAAA,IAAAA,gDAAAA,uBAAAA,AAAAA,EAAwB;YACvChB;YACA,QAAQY;QACV;QAEA,MAAMhB,YAAYC,KAAK,GAAG;QAC1B,MAAMC,SAAS,MAAM,IAAI,CAAC,mBAAmB,CAC3CpB,eACA;YACEM;QACF,GACA;YACEqB;QACF,GAEC,KAAK,CAAC,CAACY;YACNb,QAAAA,UAAAA,OAAQ,KAAK,CAAC,uBAAuBa;YACrC,MAAMA;QACR,GACC,OAAO,CAAC;YACPb,QAAAA,UAAAA,OAAQ,IAAI,CAAC,CAAC,oBAAoB,EAAEP,KAAK,GAAG,KAAKD,UAAU,EAAE,CAAC;QAChE;QAEF,IAAI,CAACE,OAAO,UAAU,EAAE;YACtB,MAAMoB,MAAM,IAAIC;YAChBD,IAAI,IAAI,GAAG;YACXA,IAAI,KAAK,GAAGX,KAAK,SAAS,CAACT,WAAW;YACtCM,QAAAA,UAAAA,OAAQ,KAAK,CAACc;YACd,MAAMA;QACR;QAEA,MAAM,EAAEE,UAAU,EAAErB,QAAQ,EAAEsB,UAAU,EAAE,GAAGvB;QAE7C,IAAI;YACF,MAAM,EAAE,QAAQwB,iBAAiB,EAAE,GAAG,MAAMC,AAAAA,IAAAA,4DAAAA,YAAAA,AAAAA,EAAa;gBACvDH;gBACA,QAAQ,IAAI,CAAC,OAAO;gBACpBlB;gBACAC;gBACA,UAAUzB;YACZ;YACA,OAAO;gBACL0C;gBACAE;gBACAvB;gBACAsB;YACF;QACF,EAAE,OAAOG,OAAO;YACdpB,QAAAA,UAAAA,OAAQ,KAAK,CAAC,uBAAuBoB;YACrC,OAAO;gBACLJ;gBACA,mBAAmB,EAAE;YACvB;QACF;IACF;IAhKA,YAA+BK,WAA8B,CAAE;QAC7D,KAAK,wDADwBA,WAAW,GAAXA;QAE7B,IAAI,CAAC,WAAW,GAAGA;IACrB;AA8JF"}