{"version":3,"file":"clickhouse.cjs","names":["VectorStore","embeddings: EmbeddingsInterface","args: ClickHouseLibArgs","vectors: number[][]","documents: Document[]","query: number[]","k: number","filter?: this[\"FilterType\"]","queryResult: {\n      data: { document: string; metadata: object; dist: number }[];\n    }","result: [Document, number][]","Document","texts: string[]","metadatas: object | object[]","docs: Document[]","dimension?: number","str: string","filter?: ClickHouseFilter","settingStrings: string[]"],"sources":["../../src/vectorstores/clickhouse.ts"],"sourcesContent":["import * as uuid from \"uuid\";\nimport { ClickHouseClient, createClient } from \"@clickhouse/client\";\nimport { format } from \"mysql2\";\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { Document } from \"@langchain/core/documents\";\n\n/**\n * Arguments for the ClickHouseStore class, which include the host, port,\n * protocol, username, password, index type, index parameters,\n * index query params, column map, database, table.\n */\nexport interface ClickHouseLibArgs {\n  host: string;\n  port: string | number;\n  protocol?: string;\n  username: string;\n  password: string;\n  indexType?: string;\n  indexParam?: string | Record<string, number>;\n  indexQueryParams?: Record<string, string>;\n  columnMap?: ColumnMap;\n  database?: string;\n  table?: string;\n}\n\n/**\n * Mapping of columns in the ClickHouse database.\n */\nexport interface ColumnMap {\n  id: string;\n  uuid: string;\n  document: string;\n  embedding: string;\n  metadata: string;\n}\n\n/**\n * Type for filtering search results in the ClickHouse database.\n */\nexport interface ClickHouseFilter {\n  whereStr: string;\n}\n\n/**\n * Class for interacting with the ClickHouse database. It extends the\n * VectorStore class and provides methods for adding vectors and\n * documents, searching for similar vectors, and creating instances from\n * texts or documents.\n */\nexport class ClickHouseStore extends VectorStore {\n  declare FilterType: ClickHouseFilter;\n\n  private client: ClickHouseClient;\n\n  private indexType: string;\n\n  private indexParam: string | Record<string, number>;\n\n  private indexQueryParams: Record<string, string>;\n\n  private columnMap: ColumnMap;\n\n  private database: string;\n\n  private table: string;\n\n  private isInitialized = false;\n\n  _vectorstoreType(): string {\n    return \"clickhouse\";\n  }\n\n  constructor(embeddings: EmbeddingsInterface, args: ClickHouseLibArgs) {\n    super(embeddings, args);\n\n    this.indexType = args.indexType || \"annoy\";\n    this.indexParam = args.indexParam || { L2Distance: 100 };\n    this.indexQueryParams = args.indexQueryParams || {};\n    this.columnMap = args.columnMap || {\n      id: \"id\",\n      document: \"document\",\n      embedding: \"embedding\",\n      metadata: \"metadata\",\n      uuid: \"uuid\",\n    };\n    this.database = args.database || \"default\";\n    this.table = args.table || \"vector_table\";\n\n    this.client = createClient({\n      host: `${args.protocol ?? \"https://\"}${args.host}:${args.port}`,\n      username: args.username,\n      password: args.password,\n      session_id: uuid.v4(),\n    });\n  }\n\n  /**\n   * Method to add vectors to the ClickHouse database.\n   * @param vectors The vectors to add.\n   * @param documents The documents associated with the vectors.\n   * @returns Promise that resolves when the vectors have been added.\n   */\n  async addVectors(vectors: number[][], documents: Document[]): Promise<void> {\n    if (vectors.length === 0) {\n      return;\n    }\n\n    if (!this.isInitialized) {\n      await this.initialize(vectors[0].length);\n    }\n\n    const queryStr = this.buildInsertQuery(vectors, documents);\n    await this.client.exec({ query: queryStr });\n  }\n\n  /**\n   * Method to add documents to the ClickHouse database.\n   * @param documents The documents to add.\n   * @returns Promise that resolves when the documents have been added.\n   */\n  async addDocuments(documents: Document[]): Promise<void> {\n    return this.addVectors(\n      await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)),\n      documents\n    );\n  }\n\n  /**\n   * Method to search for vectors that are similar to a given query vector.\n   * @param query The query vector.\n   * @param k The number of similar vectors to return.\n   * @param filter Optional filter for the search results.\n   * @returns Promise that resolves with an array of tuples, each containing a Document and a score.\n   */\n  async similaritySearchVectorWithScore(\n    query: number[],\n    k: number,\n    filter?: this[\"FilterType\"]\n  ): Promise<[Document, number][]> {\n    if (!this.isInitialized) {\n      await this.initialize(query.length);\n    }\n    const queryStr = this.buildSearchQuery(query, k, filter);\n\n    const queryResultSet = await this.client.query({ query: queryStr });\n\n    const queryResult: {\n      data: { document: string; metadata: object; dist: number }[];\n    } = await queryResultSet.json();\n\n    const result: [Document, number][] = queryResult.data.map((item) => [\n      new Document({ pageContent: item.document, metadata: item.metadata }),\n      item.dist,\n    ]);\n\n    return result;\n  }\n\n  /**\n   * Static method to create an instance of ClickHouseStore from texts.\n   * @param texts The texts to use.\n   * @param metadatas The metadata associated with the texts.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the ClickHouseStore.\n   * @returns Promise that resolves with a new instance of ClickHouseStore.\n   */\n  static async fromTexts(\n    texts: string[],\n    metadatas: object | object[],\n    embeddings: EmbeddingsInterface,\n    args: ClickHouseLibArgs\n  ): Promise<ClickHouseStore> {\n    const docs: Document[] = [];\n    for (let i = 0; i < texts.length; i += 1) {\n      const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;\n      const newDoc = new Document({\n        pageContent: texts[i],\n        metadata,\n      });\n      docs.push(newDoc);\n    }\n    return ClickHouseStore.fromDocuments(docs, embeddings, args);\n  }\n\n  /**\n   * Static method to create an instance of ClickHouseStore from documents.\n   * @param docs The documents to use.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the ClickHouseStore.\n   * @returns Promise that resolves with a new instance of ClickHouseStore.\n   */\n  static async fromDocuments(\n    docs: Document[],\n    embeddings: EmbeddingsInterface,\n    args: ClickHouseLibArgs\n  ): Promise<ClickHouseStore> {\n    const instance = new this(embeddings, args);\n    await instance.addDocuments(docs);\n    return instance;\n  }\n\n  /**\n   * Static method to create an instance of ClickHouseStore from an existing\n   * index.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the ClickHouseStore.\n   * @returns Promise that resolves with a new instance of ClickHouseStore.\n   */\n  static async fromExistingIndex(\n    embeddings: EmbeddingsInterface,\n    args: ClickHouseLibArgs\n  ): Promise<ClickHouseStore> {\n    const instance = new this(embeddings, args);\n\n    await instance.initialize();\n    return instance;\n  }\n\n  /**\n   * Method to initialize the ClickHouse database.\n   * @param dimension Optional dimension of the vectors.\n   * @returns Promise that resolves when the database has been initialized.\n   */\n  private async initialize(dimension?: number): Promise<void> {\n    const dim = dimension ?? (await this.embeddings.embedQuery(\"test\")).length;\n\n    let indexParamStr = \"\";\n    if (this.indexParam) {\n      if (typeof this.indexParam === \"string\") {\n        indexParamStr = this.indexParam;\n      } else {\n        indexParamStr = Object.entries(this.indexParam)\n          .map(([key, value]) => `'${key}', ${value}`)\n          .join(\", \");\n      }\n    }\n\n    const query = `\n    CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(\n      ${this.columnMap.id} Nullable(String),\n      ${this.columnMap.document} Nullable(String),\n      ${this.columnMap.embedding} Array(Float32),\n      ${this.columnMap.metadata} JSON,\n      ${this.columnMap.uuid} UUID DEFAULT generateUUIDv4(),\n      CONSTRAINT cons_vec_len CHECK length(${this.columnMap.embedding}) = ${dim},\n      INDEX vec_idx ${this.columnMap.embedding} TYPE ${this.indexType}(${indexParamStr}) GRANULARITY 1000\n    ) ENGINE =  MergeTree ORDER BY ${this.columnMap.uuid} SETTINGS index_granularity = 8192;`;\n\n    await this.client.exec({\n      query,\n      clickhouse_settings: {\n        allow_experimental_object_type: 1,\n        allow_experimental_annoy_index: 1,\n      },\n    });\n    this.isInitialized = true;\n  }\n\n  /**\n   * Method to build an SQL query for inserting vectors and documents into\n   * the ClickHouse database.\n   * @param vectors The vectors to insert.\n   * @param documents The documents to insert.\n   * @returns The SQL query string.\n   */\n  private buildInsertQuery(vectors: number[][], documents: Document[]): string {\n    const columnsStr = Object.values(\n      Object.fromEntries(\n        Object.entries(this.columnMap).filter(\n          ([key]) => key !== this.columnMap.uuid\n        )\n      )\n    ).join(\", \");\n\n    const placeholders = vectors.map(() => \"(?, ?, ?, ?)\").join(\", \");\n    const values = [];\n\n    for (let i = 0; i < vectors.length; i += 1) {\n      const vector = vectors[i];\n      const document = documents[i];\n      values.push(\n        uuid.v4(),\n        this.escapeString(document.pageContent),\n        JSON.stringify(vector),\n        JSON.stringify(document.metadata)\n      );\n    }\n\n    const insertQueryStr = `\n      INSERT INTO TABLE ${this.database}.${this.table}(${columnsStr}) \n      VALUES ${placeholders}\n    `;\n\n    const insertQuery = format(insertQueryStr, values);\n    return insertQuery;\n  }\n\n  private escapeString(str: string): string {\n    return str.replace(/\\\\/g, \"\\\\\\\\\").replace(/'/g, \"\\\\'\");\n  }\n\n  /**\n   * Method to build an SQL query for searching for similar vectors in the\n   * ClickHouse database.\n   * @param query The query vector.\n   * @param k The number of similar vectors to return.\n   * @param filter Optional filter for the search results.\n   * @returns The SQL query string.\n   */\n  private buildSearchQuery(\n    query: number[],\n    k: number,\n    filter?: ClickHouseFilter\n  ): string {\n    const order = \"ASC\";\n    const whereStr = filter ? `PREWHERE ${filter.whereStr}` : \"\";\n    const placeholders = query.map(() => \"?\").join(\", \");\n\n    const settingStrings: string[] = [];\n    if (this.indexQueryParams) {\n      for (const [key, value] of Object.entries(this.indexQueryParams)) {\n        settingStrings.push(`SETTING ${key}=${value}`);\n      }\n    }\n\n    const searchQueryStr = `\n      SELECT ${this.columnMap.document} AS document, ${\n        this.columnMap.metadata\n      } AS metadata, dist\n      FROM ${this.database}.${this.table}\n      ${whereStr}\n      ORDER BY L2Distance(${\n        this.columnMap.embedding\n      }, [${placeholders}]) AS dist ${order}\n      LIMIT ${k} ${settingStrings.join(\" \")}\n    `;\n\n    // Format the query with actual values\n    const searchQuery = format(searchQueryStr, query);\n    return searchQuery;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAkDA,IAAa,kBAAb,MAAa,wBAAwBA,0CAAY;CAG/C,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ,gBAAgB;CAExB,mBAA2B;AACzB,SAAO;CACR;CAED,YAAYC,YAAiCC,MAAyB;EACpE,MAAM,YAAY,KAAK;EAEvB,KAAK,YAAY,KAAK,aAAa;EACnC,KAAK,aAAa,KAAK,cAAc,EAAE,YAAY,IAAK;EACxD,KAAK,mBAAmB,KAAK,oBAAoB,CAAE;EACnD,KAAK,YAAY,KAAK,aAAa;GACjC,IAAI;GACJ,UAAU;GACV,WAAW;GACX,UAAU;GACV,MAAM;EACP;EACD,KAAK,WAAW,KAAK,YAAY;EACjC,KAAK,QAAQ,KAAK,SAAS;EAE3B,KAAK,+CAAsB;GACzB,MAAM,GAAG,KAAK,YAAY,aAAa,KAAK,KAAK,CAAC,EAAE,KAAK,MAAM;GAC/D,UAAU,KAAK;GACf,UAAU,KAAK;GACf,YAAY,KAAK,IAAI;EACtB,EAAC;CACH;;;;;;;CAQD,MAAM,WAAWC,SAAqBC,WAAsC;AAC1E,MAAI,QAAQ,WAAW,EACrB;AAGF,MAAI,CAAC,KAAK,eACR,MAAM,KAAK,WAAW,QAAQ,GAAG,OAAO;EAG1C,MAAM,WAAW,KAAK,iBAAiB,SAAS,UAAU;EAC1D,MAAM,KAAK,OAAO,KAAK,EAAE,OAAO,SAAU,EAAC;CAC5C;;;;;;CAOD,MAAM,aAAaA,WAAsC;AACvD,SAAO,KAAK,WACV,MAAM,KAAK,WAAW,eAAe,UAAU,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EACzE,UACD;CACF;;;;;;;;CASD,MAAM,gCACJC,OACAC,GACAC,QAC+B;AAC/B,MAAI,CAAC,KAAK,eACR,MAAM,KAAK,WAAW,MAAM,OAAO;EAErC,MAAM,WAAW,KAAK,iBAAiB,OAAO,GAAG,OAAO;EAExD,MAAM,iBAAiB,MAAM,KAAK,OAAO,MAAM,EAAE,OAAO,SAAU,EAAC;EAEnE,MAAMC,cAEF,MAAM,eAAe,MAAM;EAE/B,MAAMC,SAA+B,YAAY,KAAK,IAAI,CAAC,SAAS,CAClE,IAAIC,oCAAS;GAAE,aAAa,KAAK;GAAU,UAAU,KAAK;EAAU,IACpE,KAAK,IACN,EAAC;AAEF,SAAO;CACR;;;;;;;;;CAUD,aAAa,UACXC,OACAC,WACAX,YACAC,MAC0B;EAC1B,MAAMW,OAAmB,CAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,WAAW,MAAM,QAAQ,UAAU,GAAG,UAAU,KAAK;GAC3D,MAAM,SAAS,IAAIH,oCAAS;IAC1B,aAAa,MAAM;IACnB;GACD;GACD,KAAK,KAAK,OAAO;EAClB;AACD,SAAO,gBAAgB,cAAc,MAAM,YAAY,KAAK;CAC7D;;;;;;;;CASD,aAAa,cACXG,MACAZ,YACAC,MAC0B;EAC1B,MAAM,WAAW,IAAI,KAAK,YAAY;EACtC,MAAM,SAAS,aAAa,KAAK;AACjC,SAAO;CACR;;;;;;;;CASD,aAAa,kBACXD,YACAC,MAC0B;EAC1B,MAAM,WAAW,IAAI,KAAK,YAAY;EAEtC,MAAM,SAAS,YAAY;AAC3B,SAAO;CACR;;;;;;CAOD,MAAc,WAAWY,WAAmC;EAC1D,MAAM,MAAM,cAAc,MAAM,KAAK,WAAW,WAAW,OAAO,EAAE;EAEpE,IAAI,gBAAgB;AACpB,MAAI,KAAK,WACP,KAAI,OAAO,KAAK,eAAe,UAC7B,gBAAgB,KAAK;OAErB,gBAAgB,OAAO,QAAQ,KAAK,WAAW,CAC5C,IAAI,CAAC,CAAC,KAAK,MAAM,KAAK,CAAC,CAAC,EAAE,IAAI,GAAG,EAAE,OAAO,CAAC,CAC3C,KAAK,KAAK;EAIjB,MAAM,QAAQ,CAAC;+BACY,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM;MACvD,EAAE,KAAK,UAAU,GAAG;MACpB,EAAE,KAAK,UAAU,SAAS;MAC1B,EAAE,KAAK,UAAU,UAAU;MAC3B,EAAE,KAAK,UAAU,SAAS;MAC1B,EAAE,KAAK,UAAU,KAAK;2CACe,EAAE,KAAK,UAAU,UAAU,IAAI,EAAE,IAAI;oBAC5D,EAAE,KAAK,UAAU,UAAU,MAAM,EAAE,KAAK,UAAU,CAAC,EAAE,cAAc;mCACpD,EAAE,KAAK,UAAU,KAAK,mCAAmC,CAAC;EAEzF,MAAM,KAAK,OAAO,KAAK;GACrB;GACA,qBAAqB;IACnB,gCAAgC;IAChC,gCAAgC;GACjC;EACF,EAAC;EACF,KAAK,gBAAgB;CACtB;;;;;;;;CASD,AAAQ,iBAAiBX,SAAqBC,WAA+B;EAC3E,MAAM,aAAa,OAAO,OACxB,OAAO,YACL,OAAO,QAAQ,KAAK,UAAU,CAAC,OAC7B,CAAC,CAAC,IAAI,KAAK,QAAQ,KAAK,UAAU,KACnC,CACF,CACF,CAAC,KAAK,KAAK;EAEZ,MAAM,eAAe,QAAQ,IAAI,MAAM,eAAe,CAAC,KAAK,KAAK;EACjE,MAAM,SAAS,CAAE;AAEjB,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,GAAG;GAC1C,MAAM,SAAS,QAAQ;GACvB,MAAM,WAAW,UAAU;GAC3B,OAAO,KACL,KAAK,IAAI,EACT,KAAK,aAAa,SAAS,YAAY,EACvC,KAAK,UAAU,OAAO,EACtB,KAAK,UAAU,SAAS,SAAS,CAClC;EACF;EAED,MAAM,iBAAiB,CAAC;wBACJ,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,WAAW;aACvD,EAAE,aAAa;IACxB,CAAC;EAED,MAAM,iCAAqB,gBAAgB,OAAO;AAClD,SAAO;CACR;CAED,AAAQ,aAAaW,KAAqB;AACxC,SAAO,IAAI,QAAQ,OAAO,OAAO,CAAC,QAAQ,MAAM,MAAM;CACvD;;;;;;;;;CAUD,AAAQ,iBACNV,OACAC,GACAU,QACQ;EACR,MAAM,QAAQ;EACd,MAAM,WAAW,SAAS,CAAC,SAAS,EAAE,OAAO,UAAU,GAAG;EAC1D,MAAM,eAAe,MAAM,IAAI,MAAM,IAAI,CAAC,KAAK,KAAK;EAEpD,MAAMC,iBAA2B,CAAE;AACnC,MAAI,KAAK,iBACP,MAAK,MAAM,CAAC,KAAK,MAAM,IAAI,OAAO,QAAQ,KAAK,iBAAiB,EAC9D,eAAe,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC;EAIlD,MAAM,iBAAiB,CAAC;aACf,EAAE,KAAK,UAAU,SAAS,cAAc,EAC7C,KAAK,UAAU,SAChB;WACI,EAAE,KAAK,SAAS,CAAC,EAAE,KAAK,MAAM;MACnC,EAAE,SAAS;0BACS,EAClB,KAAK,UAAU,UAChB,GAAG,EAAE,aAAa,WAAW,EAAE,MAAM;YAChC,EAAE,EAAE,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACxC,CAAC;EAGD,MAAM,iCAAqB,gBAAgB,MAAM;AACjD,SAAO;CACR;AACF"}