{"version":3,"file":"base.cjs","names":["sha256","Document","uuidv5","UUIDV5_NAMESPACE"],"sources":["../../src/indexing/base.ts"],"sourcesContent":["import { v5 as uuidv5 } from \"../utils/uuid/index.js\";\nimport { VectorStore } from \"../vectorstores.js\";\nimport { RecordManagerInterface, UUIDV5_NAMESPACE } from \"./record_manager.js\";\nimport { sha256, type HashKeyEncoder } from \"../utils/hash.js\";\nimport { DocumentInterface, Document } from \"../documents/document.js\";\nimport { BaseDocumentLoader } from \"../document_loaders/base.js\";\n\ntype Metadata = Record<string, unknown>;\n\ntype IndexingResult = {\n  numAdded: number;\n  numDeleted: number;\n  numUpdated: number;\n  numSkipped: number;\n};\n\ntype StringOrDocFunc = string | ((doc: DocumentInterface) => string);\n\nexport interface HashedDocumentInterface extends DocumentInterface {\n  uid: string;\n  hash_?: string;\n  contentHash?: string;\n  metadataHash?: string;\n  pageContent: string;\n  metadata: Metadata;\n  calculateHashes(): void;\n  toDocument(): DocumentInterface;\n}\n\ninterface HashedDocumentArgs {\n  pageContent: string;\n  metadata: Metadata;\n  uid: string;\n}\n\n/**\n * HashedDocument is a Document with hashes calculated.\n * Hashes are calculated based on page content and metadata.\n * It is used for indexing.\n */\nexport class _HashedDocument implements HashedDocumentInterface {\n  uid: string;\n\n  hash_?: string;\n\n  contentHash?: string;\n\n  metadataHash?: string;\n\n  pageContent: string;\n\n  metadata: Metadata;\n\n  private keyEncoder: HashKeyEncoder = sha256;\n\n  constructor(fields: HashedDocumentArgs) {\n    this.uid = fields.uid;\n    this.pageContent = fields.pageContent;\n    this.metadata = fields.metadata;\n  }\n\n  makeDefaultKeyEncoder(keyEncoderFn: HashKeyEncoder): void {\n    this.keyEncoder = keyEncoderFn;\n  }\n\n  calculateHashes(): void {\n    const forbiddenKeys = [\"hash_\", \"content_hash\", \"metadata_hash\"];\n\n    for (const key of forbiddenKeys) {\n      if (key in this.metadata) {\n        throw new Error(\n          `Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(\n            \", \"\n          )}]`\n        );\n      }\n    }\n\n    const contentHash = this._hashStringToUUID(this.pageContent);\n\n    try {\n      const metadataHash = this._hashNestedDictToUUID(this.metadata);\n      this.contentHash = contentHash;\n      this.metadataHash = metadataHash;\n    } catch (e) {\n      throw new Error(\n        `Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`\n      );\n    }\n\n    this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);\n\n    if (!this.uid) {\n      this.uid = this.hash_;\n    }\n  }\n\n  toDocument(): DocumentInterface {\n    return new Document({\n      pageContent: this.pageContent,\n      metadata: this.metadata,\n    });\n  }\n\n  static fromDocument(\n    document: DocumentInterface,\n    uid?: string\n  ): _HashedDocument {\n    const doc = new this({\n      pageContent: document.pageContent,\n      metadata: document.metadata,\n      uid: uid || (document as DocumentInterface & { uid: string }).uid,\n    });\n    doc.calculateHashes();\n    return doc;\n  }\n\n  private _hashStringToUUID(inputString: string): string {\n    const hash_value = this.keyEncoder(inputString);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n\n  private _hashNestedDictToUUID(data: Record<string, unknown>): string {\n    const serialized_data = JSON.stringify(data, Object.keys(data).sort());\n    const hash_value = this.keyEncoder(serialized_data);\n    return uuidv5(hash_value, UUIDV5_NAMESPACE);\n  }\n}\n\nexport type CleanupMode = \"full\" | \"incremental\";\n\nexport type IndexOptions = {\n  /**\n   * The number of documents to index in one batch.\n   */\n  batchSize?: number;\n  /**\n   * The cleanup mode to use. Can be \"full\", \"incremental\" or undefined.\n   * - **Incremental**: Cleans up all documents that haven't been updated AND\n   *   that are associated with source ids that were seen\n   *   during indexing.\n   *   Clean up is done continuously during indexing helping\n   *   to minimize the probability of users seeing duplicated\n   *   content.\n   * - **Full**: Delete all documents that haven to been returned by the loader.\n   *   Clean up runs after all documents have been indexed.\n   *   This means that users may see duplicated content during indexing.\n   * - **undefined**: Do not delete any documents.\n   */\n  cleanup?: CleanupMode;\n  /**\n   * Optional key that helps identify the original source of the document.\n   * Must either be a string representing the key of the source in the metadata\n   * or a function that takes a document and returns a string representing the source.\n   * **Required when cleanup is incremental**.\n   */\n  sourceIdKey?: StringOrDocFunc;\n  /**\n   * Batch size to use when cleaning up documents.\n   */\n  cleanupBatchSize?: number;\n  /**\n   * Force update documents even if they are present in the\n   * record manager. Useful if you are re-indexing with updated embeddings.\n   */\n  forceUpdate?: boolean;\n};\n\nexport function _batch<T>(size: number, iterable: T[]): T[][] {\n  const batches: T[][] = [];\n  let currentBatch: T[] = [];\n\n  iterable.forEach((item) => {\n    currentBatch.push(item);\n\n    if (currentBatch.length >= size) {\n      batches.push(currentBatch);\n      currentBatch = [];\n    }\n  });\n\n  if (currentBatch.length > 0) {\n    batches.push(currentBatch);\n  }\n\n  return batches;\n}\n\nexport function _deduplicateInOrder(\n  hashedDocuments: HashedDocumentInterface[]\n): HashedDocumentInterface[] {\n  const seen = new Set<string>();\n  const deduplicated: HashedDocumentInterface[] = [];\n\n  for (const hashedDoc of hashedDocuments) {\n    if (!hashedDoc.hash_) {\n      throw new Error(\"Hashed document does not have a hash\");\n    }\n\n    if (!seen.has(hashedDoc.hash_)) {\n      seen.add(hashedDoc.hash_);\n      deduplicated.push(hashedDoc);\n    }\n  }\n  return deduplicated;\n}\n\nexport function _getSourceIdAssigner(\n  sourceIdKey: StringOrDocFunc | null\n): (doc: DocumentInterface) => string | null {\n  if (sourceIdKey === null) {\n    return (_doc: DocumentInterface) => null;\n  } else if (typeof sourceIdKey === \"string\") {\n    return (doc: DocumentInterface) => doc.metadata[sourceIdKey];\n  } else if (typeof sourceIdKey === \"function\") {\n    return sourceIdKey;\n  } else {\n    throw new Error(\n      `sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`\n    );\n  }\n}\n\n// oxlint-disable-next-line @typescript-eslint/no-explicit-any\nexport const _isBaseDocumentLoader = (arg: any): arg is BaseDocumentLoader => {\n  if (\n    \"load\" in arg &&\n    typeof arg.load === \"function\" &&\n    \"loadAndSplit\" in arg &&\n    typeof arg.loadAndSplit === \"function\"\n  ) {\n    return true;\n  }\n  return false;\n};\n\ninterface IndexArgs {\n  docsSource: BaseDocumentLoader | DocumentInterface[];\n  recordManager: RecordManagerInterface;\n  vectorStore: VectorStore;\n  options?: IndexOptions;\n}\n\n/**\n * Index data from the doc source into the vector store.\n *\n * Indexing functionality uses a manager to keep track of which documents\n * are in the vector store.\n *\n * This allows us to keep track of which documents were updated, and which\n * documents were deleted, which documents should be skipped.\n *\n * For the time being, documents are indexed using their hashes, and users\n *  are not able to specify the uid of the document.\n *\n * @param {IndexArgs} args\n * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.\n * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.\n * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.\n * @param {IndexOptions | undefined} args.options Options for indexing.\n * @returns {Promise<IndexingResult>}\n */\nexport async function index(args: IndexArgs): Promise<IndexingResult> {\n  const { docsSource, recordManager, vectorStore, options } = args;\n  const {\n    batchSize = 100,\n    cleanup,\n    sourceIdKey,\n    cleanupBatchSize = 1000,\n    forceUpdate = false,\n  } = options ?? {};\n\n  if (cleanup === \"incremental\" && !sourceIdKey) {\n    throw new Error(\n      \"sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.\"\n    );\n  }\n\n  const docs = _isBaseDocumentLoader(docsSource)\n    ? await docsSource.load()\n    : docsSource;\n\n  const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);\n\n  const indexStartDt = await recordManager.getTime();\n  let numAdded = 0;\n  let numDeleted = 0;\n  let numUpdated = 0;\n  let numSkipped = 0;\n\n  const batches = _batch<DocumentInterface>(batchSize ?? 100, docs);\n\n  for (const batch of batches) {\n    const hashedDocs = _deduplicateInOrder(\n      batch.map((doc) => _HashedDocument.fromDocument(doc))\n    );\n\n    const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));\n\n    if (cleanup === \"incremental\") {\n      hashedDocs.forEach((_hashedDoc, index) => {\n        const source = sourceIds[index];\n        if (source === null) {\n          throw new Error(\n            \"sourceIdKey must be provided when cleanup is incremental\"\n          );\n        }\n      });\n    }\n\n    const batchExists = await recordManager.exists(\n      hashedDocs.map((doc) => doc.uid)\n    );\n\n    const uids: string[] = [];\n    const docsToIndex: DocumentInterface[] = [];\n    const docsToUpdate: string[] = [];\n    const seenDocs = new Set<string>();\n    hashedDocs.forEach((hashedDoc, i) => {\n      const docExists = batchExists[i];\n      if (docExists) {\n        if (forceUpdate) {\n          seenDocs.add(hashedDoc.uid);\n        } else {\n          docsToUpdate.push(hashedDoc.uid);\n          return;\n        }\n      }\n      uids.push(hashedDoc.uid);\n      docsToIndex.push(hashedDoc.toDocument());\n    });\n\n    if (docsToUpdate.length > 0) {\n      await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });\n      numSkipped += docsToUpdate.length;\n    }\n\n    if (docsToIndex.length > 0) {\n      await vectorStore.addDocuments(docsToIndex, { ids: uids });\n      numAdded += docsToIndex.length - seenDocs.size;\n      numUpdated += seenDocs.size;\n    }\n\n    await recordManager.update(\n      hashedDocs.map((doc) => doc.uid),\n      { timeAtLeast: indexStartDt, groupIds: sourceIds }\n    );\n\n    if (cleanup === \"incremental\") {\n      sourceIds.forEach((sourceId) => {\n        if (!sourceId) throw new Error(\"Source id cannot be null\");\n      });\n      const uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        groupIds: sourceIds,\n      });\n\n      if (uidsToDelete.length > 0) {\n        await vectorStore.delete({ ids: uidsToDelete });\n        await recordManager.deleteKeys(uidsToDelete);\n        numDeleted += uidsToDelete.length;\n      }\n    }\n  }\n\n  if (cleanup === \"full\") {\n    let uidsToDelete = await recordManager.listKeys({\n      before: indexStartDt,\n      limit: cleanupBatchSize,\n    });\n    while (uidsToDelete.length > 0) {\n      await vectorStore.delete({ ids: uidsToDelete });\n      await recordManager.deleteKeys(uidsToDelete);\n      numDeleted += uidsToDelete.length;\n      uidsToDelete = await recordManager.listKeys({\n        before: indexStartDt,\n        limit: cleanupBatchSize,\n      });\n    }\n  }\n\n  return {\n    numAdded,\n    numDeleted,\n    numUpdated,\n    numSkipped,\n  };\n}\n"],"mappings":";;;;;;;;;;;AAwCA,IAAa,kBAAb,MAAgE;CAC9D;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,aAAqCA,aAAAA;CAErC,YAAY,QAA4B;EACtC,KAAK,MAAM,OAAO;EAClB,KAAK,cAAc,OAAO;EAC1B,KAAK,WAAW,OAAO;CACzB;CAEA,sBAAsB,cAAoC;EACxD,KAAK,aAAa;CACpB;CAEA,kBAAwB;EACtB,MAAM,gBAAgB;GAAC;GAAS;GAAgB;EAAe;EAE/D,KAAK,MAAM,OAAO,eAChB,IAAI,OAAO,KAAK,UACd,MAAM,IAAI,MACR,+BAA+B,IAAI,yDAAyD,cAAc,KACxG,IACF,EAAE,EACJ;EAIJ,MAAM,cAAc,KAAK,kBAAkB,KAAK,WAAW;EAE3D,IAAI;GACF,MAAM,eAAe,KAAK,sBAAsB,KAAK,QAAQ;GAC7D,KAAK,cAAc;GACnB,KAAK,eAAe;EACtB,SAAS,GAAG;GACV,MAAM,IAAI,MACR,4BAA4B,EAAE,uDAChC;EACF;EAEA,KAAK,QAAQ,KAAK,kBAAkB,KAAK,cAAc,KAAK,YAAY;EAExE,IAAI,CAAC,KAAK,KACR,KAAK,MAAM,KAAK;CAEpB;CAEA,aAAgC;EAC9B,OAAO,IAAIC,iBAAAA,SAAS;GAClB,aAAa,KAAK;GAClB,UAAU,KAAK;EACjB,CAAC;CACH;CAEA,OAAO,aACL,UACA,KACiB;EACjB,MAAM,MAAM,IAAI,KAAK;GACnB,aAAa,SAAS;GACtB,UAAU,SAAS;GACnB,KAAK,OAAQ,SAAiD;EAChE,CAAC;EACD,IAAI,gBAAgB;EACpB,OAAO;CACT;CAEA,kBAA0B,aAA6B;EAErD,OAAOC,yBAAAA,GADY,KAAK,WAAW,WACZ,GAAGC,uBAAAA,gBAAgB;CAC5C;CAEA,sBAA8B,MAAuC;EACnE,MAAM,kBAAkB,KAAK,UAAU,MAAM,OAAO,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC;EAErE,OAAOD,yBAAAA,GADY,KAAK,WAAW,eACZ,GAAGC,uBAAAA,gBAAgB;CAC5C;AACF;AAyCA,SAAgB,OAAU,MAAc,UAAsB;CAC5D,MAAM,UAAiB,CAAC;CACxB,IAAI,eAAoB,CAAC;CAEzB,SAAS,SAAS,SAAS;EACzB,aAAa,KAAK,IAAI;EAEtB,IAAI,aAAa,UAAU,MAAM;GAC/B,QAAQ,KAAK,YAAY;GACzB,eAAe,CAAC;EAClB;CACF,CAAC;CAED,IAAI,aAAa,SAAS,GACxB,QAAQ,KAAK,YAAY;CAG3B,OAAO;AACT;AAEA,SAAgB,oBACd,iBAC2B;CAC3B,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,eAA0C,CAAC;CAEjD,KAAK,MAAM,aAAa,iBAAiB;EACvC,IAAI,CAAC,UAAU,OACb,MAAM,IAAI,MAAM,sCAAsC;EAGxD,IAAI,CAAC,KAAK,IAAI,UAAU,KAAK,GAAG;GAC9B,KAAK,IAAI,UAAU,KAAK;GACxB,aAAa,KAAK,SAAS;EAC7B;CACF;CACA,OAAO;AACT;AAEA,SAAgB,qBACd,aAC2C;CAC3C,IAAI,gBAAgB,MAClB,QAAQ,SAA4B;MAC/B,IAAI,OAAO,gBAAgB,UAChC,QAAQ,QAA2B,IAAI,SAAS;MAC3C,IAAI,OAAO,gBAAgB,YAChC,OAAO;MAEP,MAAM,IAAI,MACR,2DAA2D,OAAO,aACpE;AAEJ;AAGA,MAAa,yBAAyB,QAAwC;CAC5E,IACE,UAAU,OACV,OAAO,IAAI,SAAS,cACpB,kBAAkB,OAClB,OAAO,IAAI,iBAAiB,YAE5B,OAAO;CAET,OAAO;AACT;;;;;;;;;;;;;;;;;;;;AA4BA,eAAsB,MAAM,MAA0C;CACpE,MAAM,EAAE,YAAY,eAAe,aAAa,YAAY;CAC5D,MAAM,EACJ,YAAY,KACZ,SACA,aACA,mBAAmB,KACnB,cAAc,UACZ,WAAW,CAAC;CAEhB,IAAI,YAAY,iBAAiB,CAAC,aAChC,MAAM,IAAI,MACR,yGACF;CAGF,MAAM,OAAO,sBAAsB,UAAU,IACzC,MAAM,WAAW,KAAK,IACtB;CAEJ,MAAM,mBAAmB,qBAAqB,eAAe,IAAI;CAEjE,MAAM,eAAe,MAAM,cAAc,QAAQ;CACjD,IAAI,WAAW;CACf,IAAI,aAAa;CACjB,IAAI,aAAa;CACjB,IAAI,aAAa;CAEjB,MAAM,UAAU,OAA0B,aAAa,KAAK,IAAI;CAEhE,KAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,aAAa,oBACjB,MAAM,KAAK,QAAQ,gBAAgB,aAAa,GAAG,CAAC,CACtD;EAEA,MAAM,YAAY,WAAW,KAAK,QAAQ,iBAAiB,GAAG,CAAC;EAE/D,IAAI,YAAY,eACd,WAAW,SAAS,YAAY,UAAU;GAExC,IADe,UAAU,WACV,MACb,MAAM,IAAI,MACR,0DACF;EAEJ,CAAC;EAGH,MAAM,cAAc,MAAM,cAAc,OACtC,WAAW,KAAK,QAAQ,IAAI,GAAG,CACjC;EAEA,MAAM,OAAiB,CAAC;EACxB,MAAM,cAAmC,CAAC;EAC1C,MAAM,eAAyB,CAAC;EAChC,MAAM,2BAAW,IAAI,IAAY;EACjC,WAAW,SAAS,WAAW,MAAM;GAEnC,IADkB,YAAY,IAE5B,IAAI,aACF,SAAS,IAAI,UAAU,GAAG;QACrB;IACL,aAAa,KAAK,UAAU,GAAG;IAC/B;GACF;GAEF,KAAK,KAAK,UAAU,GAAG;GACvB,YAAY,KAAK,UAAU,WAAW,CAAC;EACzC,CAAC;EAED,IAAI,aAAa,SAAS,GAAG;GAC3B,MAAM,cAAc,OAAO,cAAc,EAAE,aAAa,aAAa,CAAC;GACtE,cAAc,aAAa;EAC7B;EAEA,IAAI,YAAY,SAAS,GAAG;GAC1B,MAAM,YAAY,aAAa,aAAa,EAAE,KAAK,KAAK,CAAC;GACzD,YAAY,YAAY,SAAS,SAAS;GAC1C,cAAc,SAAS;EACzB;EAEA,MAAM,cAAc,OAClB,WAAW,KAAK,QAAQ,IAAI,GAAG,GAC/B;GAAE,aAAa;GAAc,UAAU;EAAU,CACnD;EAEA,IAAI,YAAY,eAAe;GAC7B,UAAU,SAAS,aAAa;IAC9B,IAAI,CAAC,UAAU,MAAM,IAAI,MAAM,0BAA0B;GAC3D,CAAC;GACD,MAAM,eAAe,MAAM,cAAc,SAAS;IAChD,QAAQ;IACR,UAAU;GACZ,CAAC;GAED,IAAI,aAAa,SAAS,GAAG;IAC3B,MAAM,YAAY,OAAO,EAAE,KAAK,aAAa,CAAC;IAC9C,MAAM,cAAc,WAAW,YAAY;IAC3C,cAAc,aAAa;GAC7B;EACF;CACF;CAEA,IAAI,YAAY,QAAQ;EACtB,IAAI,eAAe,MAAM,cAAc,SAAS;GAC9C,QAAQ;GACR,OAAO;EACT,CAAC;EACD,OAAO,aAAa,SAAS,GAAG;GAC9B,MAAM,YAAY,OAAO,EAAE,KAAK,aAAa,CAAC;GAC9C,MAAM,cAAc,WAAW,YAAY;GAC3C,cAAc,aAAa;GAC3B,eAAe,MAAM,cAAc,SAAS;IAC1C,QAAQ;IACR,OAAO;GACT,CAAC;EACH;CACF;CAEA,OAAO;EACL;EACA;EACA;EACA;CACF;AACF"}