{"version":3,"file":"myscale.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","args: MyScaleLibArgs","vectors: number[][]","documents: Document[]","query: number[]","k: number","filter?: this[\"FilterType\"]","queryResult: {\n      data: { text: string; metadata: object; dist: number }[];\n    }","result: [Document, number][]","Document","texts: string[]","metadatas: object | object[]","docs: Document[]","dimension?: number","data: string[]","str: string","filter?: MyScaleFilter"],"sources":["../../src/vectorstores/myscale.ts"],"sourcesContent":["import * as uuid from \"uuid\";\nimport { ClickHouseClient, createClient } from \"@clickhouse/client\";\n\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { Document } from \"@langchain/core/documents\";\n\n/**\n * Arguments for the MyScaleStore class, which include the host, port,\n * protocol, username, password, index type, index parameters, column map,\n * database, table, and metric.\n */\nexport interface MyScaleLibArgs {\n  host: string;\n  port: string | number;\n  protocol?: string;\n  username: string;\n  password: string;\n  indexType?: string;\n  indexParam?: Record<string, string>;\n  columnMap?: ColumnMap;\n  database?: string;\n  table?: string;\n  metric?: metric;\n}\n\n/**\n * Mapping of columns in the MyScale database.\n */\nexport interface ColumnMap {\n  id: string;\n  text: string;\n  vector: string;\n  metadata: string;\n}\n\n/**\n * Type of metric used in the MyScale database.\n */\nexport type metric = \"L2\" | \"Cosine\" | \"IP\";\n\n/**\n * Type for filtering search results in the MyScale database.\n */\nexport interface MyScaleFilter {\n  whereStr: string;\n}\n\n/**\n * Class for interacting with the MyScale database. It extends the\n * VectorStore class and provides methods for adding vectors and\n * documents, searching for similar vectors, and creating instances from\n * texts or documents.\n */\nexport class MyScaleStore extends VectorStore {\n  declare FilterType: MyScaleFilter;\n\n  private client: ClickHouseClient;\n\n  private indexType: string;\n\n  private indexParam: Record<string, string>;\n\n  private columnMap: ColumnMap;\n\n  private database: string;\n\n  private table: string;\n\n  private metric: metric;\n\n  private isInitialized = false;\n\n  _vectorstoreType(): string {\n    return \"myscale\";\n  }\n\n  constructor(embeddings: EmbeddingsInterface, args: MyScaleLibArgs) {\n    super(embeddings, args);\n\n    this.indexType = args.indexType || \"MSTG\";\n    this.indexParam = args.indexParam || {};\n    this.columnMap = args.columnMap || {\n      id: \"id\",\n      text: \"text\",\n      vector: \"vector\",\n      metadata: \"metadata\",\n    };\n    this.database = args.database || \"default\";\n    this.table = args.table || \"vector_table\";\n    this.metric = args.metric || \"Cosine\";\n\n    this.client = createClient({\n      host: `${args.protocol ?? \"https://\"}${args.host}:${args.port}`,\n      username: args.username,\n      password: args.password,\n      session_id: uuid.v4(),\n    });\n  }\n\n  /**\n   * Method to add vectors to the MyScale database.\n   * @param vectors The vectors to add.\n   * @param documents The documents associated with the vectors.\n   * @returns Promise that resolves when the vectors have been added.\n   */\n  async addVectors(vectors: number[][], documents: Document[]): Promise<void> {\n    if (vectors.length === 0) {\n      return;\n    }\n\n    if (!this.isInitialized) {\n      await this.initialize(vectors[0].length);\n    }\n\n    const queryStr = this.buildInsertQuery(vectors, documents);\n    await this.client.exec({ query: queryStr });\n  }\n\n  /**\n   * Method to add documents to the MyScale database.\n   * @param documents The documents to add.\n   * @returns Promise that resolves when the documents have been added.\n   */\n  async addDocuments(documents: Document[]): Promise<void> {\n    return this.addVectors(\n      await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)),\n      documents\n    );\n  }\n\n  /**\n   * Method to search for vectors that are similar to a given query vector.\n   * @param query The query vector.\n   * @param k The number of similar vectors to return.\n   * @param filter Optional filter for the search results.\n   * @returns Promise that resolves with an array of tuples, each containing a Document and a score.\n   */\n  async similaritySearchVectorWithScore(\n    query: number[],\n    k: number,\n    filter?: this[\"FilterType\"]\n  ): Promise<[Document, number][]> {\n    if (!this.isInitialized) {\n      await this.initialize(query.length);\n    }\n    const queryStr = this.buildSearchQuery(query, k, filter);\n\n    const queryResultSet = await this.client.query({ query: queryStr });\n    const queryResult: {\n      data: { text: string; metadata: object; dist: number }[];\n    } = await queryResultSet.json();\n\n    const result: [Document, number][] = queryResult.data.map((item) => [\n      new Document({ pageContent: item.text, metadata: item.metadata }),\n      item.dist,\n    ]);\n\n    return result;\n  }\n\n  /**\n   * Static method to create an instance of MyScaleStore from texts.\n   * @param texts The texts to use.\n   * @param metadatas The metadata associated with the texts.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the MyScaleStore.\n   * @returns Promise that resolves with a new instance of MyScaleStore.\n   */\n  static async fromTexts(\n    texts: string[],\n    metadatas: object | object[],\n    embeddings: EmbeddingsInterface,\n    args: MyScaleLibArgs\n  ): Promise<MyScaleStore> {\n    const docs: Document[] = [];\n    for (let i = 0; i < texts.length; i += 1) {\n      const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;\n      const newDoc = new Document({\n        pageContent: texts[i],\n        metadata,\n      });\n      docs.push(newDoc);\n    }\n    return MyScaleStore.fromDocuments(docs, embeddings, args);\n  }\n\n  /**\n   * Static method to create an instance of MyScaleStore from documents.\n   * @param docs The documents to use.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the MyScaleStore.\n   * @returns Promise that resolves with a new instance of MyScaleStore.\n   */\n  static async fromDocuments(\n    docs: Document[],\n    embeddings: EmbeddingsInterface,\n    args: MyScaleLibArgs\n  ): Promise<MyScaleStore> {\n    const instance = new this(embeddings, args);\n    await instance.addDocuments(docs);\n    return instance;\n  }\n\n  /**\n   * Static method to create an instance of MyScaleStore from an existing\n   * index.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the MyScaleStore.\n   * @returns Promise that resolves with a new instance of MyScaleStore.\n   */\n  static async fromExistingIndex(\n    embeddings: EmbeddingsInterface,\n    args: MyScaleLibArgs\n  ): Promise<MyScaleStore> {\n    const instance = new this(embeddings, args);\n\n    await instance.initialize();\n    return instance;\n  }\n\n  /**\n   * Method to initialize the MyScale database.\n   * @param dimension Optional dimension of the vectors.\n   * @returns Promise that resolves when the database has been initialized.\n   */\n  private async initialize(dimension?: number): Promise<void> {\n    const dim = dimension ?? (await this.embeddings.embedQuery(\"test\")).length;\n\n    let indexParamStr = \"\";\n    for (const [key, value] of Object.entries(this.indexParam)) {\n      indexParamStr += `, '${key}=${value}'`;\n    }\n\n    const query = `\n      CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(\n        ${this.columnMap.id} String,\n        ${this.columnMap.text} String,\n        ${this.columnMap.vector} Array(Float32),\n        ${this.columnMap.metadata} JSON,\n        CONSTRAINT cons_vec_len CHECK length(${this.columnMap.vector}) = ${dim},\n        VECTOR INDEX vidx ${this.columnMap.vector} TYPE ${this.indexType}('metric_type=${this.metric}'${indexParamStr})\n      ) ENGINE = MergeTree ORDER BY ${this.columnMap.id}\n    `;\n\n    await this.client.exec({ query: \"SET allow_experimental_object_type=1\" });\n    await this.client.exec({\n      query: \"SET output_format_json_named_tuples_as_objects = 1\",\n    });\n    await this.client.exec({ query });\n    this.isInitialized = true;\n  }\n\n  /**\n   * Method to build an SQL query for inserting vectors and documents into\n   * the MyScale database.\n   * @param vectors The vectors to insert.\n   * @param documents The documents to insert.\n   * @returns The SQL query string.\n   */\n  private buildInsertQuery(vectors: number[][], documents: Document[]): string {\n    const columnsStr = Object.values(this.columnMap).join(\", \");\n\n    const data: string[] = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      const vector = vectors[i];\n      const document = documents[i];\n      const item = [\n        `'${uuid.v4()}'`,\n        `'${this.escapeString(document.pageContent)}'`,\n        `[${vector}]`,\n        `'${JSON.stringify(document.metadata)}'`,\n      ].join(\", \");\n      data.push(`(${item})`);\n    }\n    const dataStr = data.join(\", \");\n\n    return `\n      INSERT INTO TABLE\n        ${this.database}.${this.table}(${columnsStr})\n      VALUES\n        ${dataStr}\n    `;\n  }\n\n  private escapeString(str: string): string {\n    return str.replace(/\\\\/g, \"\\\\\\\\\").replace(/'/g, \"\\\\'\");\n  }\n\n  /**\n   * Method to build an SQL query for searching for similar vectors in the\n   * MyScale database.\n   * @param query The query vector.\n   * @param k The number of similar vectors to return.\n   * @param filter Optional filter for the search results.\n   * @returns The SQL query string.\n   */\n  private buildSearchQuery(\n    query: number[],\n    k: number,\n    filter?: MyScaleFilter\n  ): string {\n    const order = this.metric === \"IP\" ? \"DESC\" : \"ASC\";\n\n    const whereStr = filter ? `PREWHERE ${filter.whereStr}` : \"\";\n    return `\n      SELECT ${this.columnMap.text} AS text, ${this.columnMap.metadata} AS metadata, dist\n      FROM ${this.database}.${this.table}\n      ${whereStr}\n      ORDER BY distance(${this.columnMap.vector}, [${query}]) AS dist ${order}\n      LIMIT ${k}\n    `;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAsDA,IAAa,eAAb,MAAa,qBAAqBA,0CAAY;CAG5C,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ,gBAAgB;CAExB,mBAA2B;AACzB,SAAO;CACR;CAED,YAAYC,YAAiCC,MAAsB;EACjE,MAAM,YAAY,KAAK;EAEvB,KAAK,YAAY,KAAK,aAAa;EACnC,KAAK,aAAa,KAAK,cAAc,CAAE;EACvC,KAAK,YAAY,KAAK,aAAa;GACjC,IAAI;GACJ,MAAM;GACN,QAAQ;GACR,UAAU;EACX;EACD,KAAK,WAAW,KAAK,YAAY;EACjC,KAAK,QAAQ,KAAK,SAAS;EAC3B,KAAK,SAAS,KAAK,UAAU;EAE7B,KAAK,+CAAsB;GACzB,MAAM,GAAG,KAAK,YAAY,aAAa,KAAK,KAAK,CAAC,EAAE,KAAK,MAAM;GAC/D,UAAU,KAAK;GACf,UAAU,KAAK;GACf,YAAY,KAAK,IAAI;EACtB,EAAC;CACH;;;;;;;CAQD,MAAM,WAAWC,SAAqBC,WAAsC;AAC1E,MAAI,QAAQ,WAAW,EACrB;AAGF,MAAI,CAAC,KAAK,eACR,MAAM,KAAK,WAAW,QAAQ,GAAG,OAAO;EAG1C,MAAM,WAAW,KAAK,iBAAiB,SAAS,UAAU;EAC1D,MAAM,KAAK,OAAO,KAAK,EAAE,OAAO,SAAU,EAAC;CAC5C;;;;;;CAOD,MAAM,aAAaA,WAAsC;AACvD,SAAO,KAAK,WACV,MAAM,KAAK,WAAW,eAAe,UAAU,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EACzE,UACD;CACF;;;;;;;;CASD,MAAM,gCACJC,OACAC,GACAC,QAC+B;AAC/B,MAAI,CAAC,KAAK,eACR,MAAM,KAAK,WAAW,MAAM,OAAO;EAErC,MAAM,WAAW,KAAK,iBAAiB,OAAO,GAAG,OAAO;EAExD,MAAM,iBAAiB,MAAM,KAAK,OAAO,MAAM,EAAE,OAAO,SAAU,EAAC;EACnE,MAAMC,cAEF,MAAM,eAAe,MAAM;EAE/B,MAAMC,SAA+B,YAAY,KAAK,IAAI,CAAC,SAAS,CAClE,IAAIC,oCAAS;GAAE,aAAa,KAAK;GAAM,UAAU,KAAK;EAAU,IAChE,KAAK,IACN,EAAC;AAEF,SAAO;CACR;;;;;;;;;CAUD,aAAa,UACXC,OACAC,WACAX,YACAC,MACuB;EACvB,MAAMW,OAAmB,CAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,WAAW,MAAM,QAAQ,UAAU,GAAG,UAAU,KAAK;GAC3D,MAAM,SAAS,IAAIH,oCAAS;IAC1B,aAAa,MAAM;IACnB;GACD;GACD,KAAK,KAAK,OAAO;EAClB;AACD,SAAO,aAAa,cAAc,MAAM,YAAY,KAAK;CAC1D;;;;;;;;CASD,aAAa,cACXG,MACAZ,YACAC,MACuB;EACvB,MAAM,WAAW,IAAI,KAAK,YAAY;EACtC,MAAM,SAAS,aAAa,KAAK;AACjC,SAAO;CACR;;;;;;;;CASD,aAAa,kBACXD,YACAC,MACuB;EACvB,MAAM,WAAW,IAAI,KAAK,YAAY;EAEtC,MAAM,SAAS,YAAY;AAC3B,SAAO;CACR;;;;;;CAOD,MAAc,WAAWY,WAAmC;EAC1D,MAAM,MAAM,cAAc,MAAM,KAAK,WAAW,WAAW,OAAO,EAAE;EAEpE,IAAI,gBAAgB;AACpB,OAAK,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,QAAQ,KAAK,WAAW,EACxD,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;EAGxC,MAAM,QAAQ,CAAC;iCACc,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM;QACvD,EAAE,KAAK,UAAU,GAAG;QACpB,EAAE,KAAK,UAAU,KAAK;QACtB,EAAE,KAAK,UAAU,OAAO;QACxB,EAAE,KAAK,UAAU,SAAS;6CACW,EAAE,KAAK,UAAU,OAAO,IAAI,EAAE,IAAI;0BACrD,EAAE,KAAK,UAAU,OAAO,MAAM,EAAE,KAAK,UAAU,cAAc,EAAE,KAAK,OAAO,CAAC,EAAE,cAAc;oCAClF,EAAE,KAAK,UAAU,GAAG;IACpD,CAAC;EAED,MAAM,KAAK,OAAO,KAAK,EAAE,OAAO,uCAAwC,EAAC;EACzE,MAAM,KAAK,OAAO,KAAK,EACrB,OAAO,qDACR,EAAC;EACF,MAAM,KAAK,OAAO,KAAK,EAAE,MAAO,EAAC;EACjC,KAAK,gBAAgB;CACtB;;;;;;;;CASD,AAAQ,iBAAiBX,SAAqBC,WAA+B;EAC3E,MAAM,aAAa,OAAO,OAAO,KAAK,UAAU,CAAC,KAAK,KAAK;EAE3D,MAAMW,OAAiB,CAAE;AACzB,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,GAAG;GAC1C,MAAM,SAAS,QAAQ;GACvB,MAAM,WAAW,UAAU;GAC3B,MAAM,OAAO;IACX,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;IAChB,CAAC,CAAC,EAAE,KAAK,aAAa,SAAS,YAAY,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACb,CAAC,CAAC,EAAE,KAAK,UAAU,SAAS,SAAS,CAAC,CAAC,CAAC;GACzC,EAAC,KAAK,KAAK;GACZ,KAAK,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;EACvB;EACD,MAAM,UAAU,KAAK,KAAK,KAAK;AAE/B,SAAO,CAAC;;QAEJ,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,WAAW;;QAE5C,EAAE,QAAQ;IACd,CAAC;CACF;CAED,AAAQ,aAAaC,KAAqB;AACxC,SAAO,IAAI,QAAQ,OAAO,OAAO,CAAC,QAAQ,MAAM,MAAM;CACvD;;;;;;;;;CAUD,AAAQ,iBACNX,OACAC,GACAW,QACQ;EACR,MAAM,QAAQ,KAAK,WAAW,OAAO,SAAS;EAE9C,MAAM,WAAW,SAAS,CAAC,SAAS,EAAE,OAAO,UAAU,GAAG;AAC1D,SAAO,CAAC;aACC,EAAE,KAAK,UAAU,KAAK,UAAU,EAAE,KAAK,UAAU,SAAS;WAC5D,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM;MACnC,EAAE,SAAS;wBACO,EAAE,KAAK,UAAU,OAAO,GAAG,EAAE,MAAM,WAAW,EAAE,MAAM;YAClE,EAAE,EAAE;IACZ,CAAC;CACF;AACF"}