{"version":3,"file":"cheerio.cjs","names":["BaseDocumentLoader","webPath: string","fields?: CheerioWebBaseLoaderParams","AsyncCaller","urls: string[]","caller: AsyncCaller","timeout: number | undefined","textDecoder?: TextDecoder","options?: CheerioOptions & {\n headers?: HeadersInit;\n }","url: string","Document"],"sources":["../../../src/document_loaders/web/cheerio.ts"],"sourcesContent":["import type {\n CheerioAPI,\n CheerioOptions,\n load as LoadT,\n SelectorType,\n} from \"cheerio\";\nimport { Document } from \"@langchain/core/documents\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\nimport type { WebBaseLoaderParams, WebBaseLoader } from \"./html.js\";\n\n/**\n * Represents the parameters for configuring the CheerioWebBaseLoader. It\n * extends the WebBaseLoaderParams interface and adds additional parameters\n * specific to loading with Cheerio.\n */\nexport interface CheerioWebBaseLoaderParams extends WebBaseLoaderParams {\n /**\n * The selector to use to extract the text from the document. Defaults to\n * \"body\".\n */\n selector?: SelectorType;\n}\n\n/**\n * A class that extends the BaseDocumentLoader and implements the\n * DocumentLoader interface. It represents a document loader for loading\n * web-based documents using Cheerio.\n * @example\n * ```typescript\n * const loader = new CheerioWebBaseLoader(\"https://exampleurl.com\");\n * const docs = await loader.load();\n * console.log({ docs });\n * ```\n */\nexport class CheerioWebBaseLoader\n extends BaseDocumentLoader\n implements WebBaseLoader\n{\n timeout: number;\n\n caller: AsyncCaller;\n\n selector?: SelectorType;\n\n textDecoder?: TextDecoder;\n\n headers?: HeadersInit;\n\n constructor(\n public webPath: string,\n fields?: CheerioWebBaseLoaderParams\n ) {\n super();\n const { timeout, selector, textDecoder, headers, ...rest } = fields ?? {};\n this.timeout = timeout ?? 10000;\n this.caller = new AsyncCaller(rest);\n this.selector = selector ?? \"body\";\n this.textDecoder = textDecoder;\n this.headers = headers;\n }\n\n /**\n * Fetches web documents from the given array of URLs and loads them using Cheerio.\n * It returns an array of CheerioAPI instances.\n * @param urls An array of URLs to fetch and load.\n * @returns A Promise that resolves to an array of CheerioAPI instances.\n */\n static async scrapeAll(\n urls: string[],\n caller: AsyncCaller,\n timeout: number | undefined,\n textDecoder?: TextDecoder,\n options?: CheerioOptions & {\n headers?: HeadersInit;\n }\n ): Promise {\n return Promise.all(\n urls.map((url) =>\n CheerioWebBaseLoader._scrape(url, caller, timeout, textDecoder, options)\n )\n );\n }\n\n static async _scrape(\n url: string,\n caller: AsyncCaller,\n timeout: number | undefined,\n textDecoder?: TextDecoder,\n options?: CheerioOptions & {\n headers?: HeadersInit;\n }\n ): Promise {\n const { headers, ...cheerioOptions } = options ?? {};\n const { load } = await CheerioWebBaseLoader.imports();\n const response = await caller.call(fetch, url, {\n signal: timeout ? AbortSignal.timeout(timeout) : undefined,\n headers,\n });\n const html =\n textDecoder?.decode(await response.arrayBuffer()) ??\n (await response.text());\n return load(html, cheerioOptions);\n }\n\n /**\n * Fetches the web document from the webPath and loads it using Cheerio.\n * It returns a CheerioAPI instance.\n * @returns A Promise that resolves to a CheerioAPI instance.\n */\n async scrape(): Promise {\n const options = { headers: this.headers };\n return CheerioWebBaseLoader._scrape(\n this.webPath,\n this.caller,\n this.timeout,\n this.textDecoder,\n options\n );\n }\n\n /**\n * Extracts the text content from the loaded document using the selector\n * and creates a Document instance with the extracted text and metadata.\n * It returns an array of Document instances.\n * @returns A Promise that resolves to an array of Document instances.\n */\n async load(): Promise {\n const $ = await this.scrape();\n const title = $(\"title\").text();\n const text = $(this.selector).text();\n const metadata = { source: this.webPath, title };\n return [new Document({ pageContent: text, metadata })];\n }\n\n /**\n * A static method that dynamically imports the Cheerio library and\n * returns the load function. If the import fails, it throws an error.\n * @returns A Promise that resolves to an object containing the load function from the Cheerio library.\n */\n static async imports(): Promise<{\n load: typeof LoadT;\n }> {\n try {\n const { load } = await import(\"cheerio\");\n return { load };\n } catch (e) {\n console.error(e);\n throw new Error(\n \"Please install cheerio as a dependency with, e.g. `pnpm install cheerio`\"\n );\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAmCA,IAAa,uBAAb,MAAa,6BACHA,0DAEV;CACE;CAEA;CAEA;CAEA;CAEA;CAEA,YACSC,SACPC,QACA;EACA,OAAO;EAHA;EAIP,MAAM,EAAE,SAAS,UAAU,aAAa,QAAS,GAAG,MAAM,GAAG,UAAU,CAAE;EACzE,KAAK,UAAU,WAAW;EAC1B,KAAK,SAAS,IAAIC,gDAAY;EAC9B,KAAK,WAAW,YAAY;EAC5B,KAAK,cAAc;EACnB,KAAK,UAAU;CAChB;;;;;;;CAQD,aAAa,UACXC,MACAC,QACAC,SACAC,aACAC,SAGuB;AACvB,SAAO,QAAQ,IACb,KAAK,IAAI,CAAC,QACR,qBAAqB,QAAQ,KAAK,QAAQ,SAAS,aAAa,QAAQ,CACzE,CACF;CACF;CAED,aAAa,QACXC,KACAJ,QACAC,SACAC,aACAC,SAGqB;EACrB,MAAM,EAAE,QAAS,GAAG,gBAAgB,GAAG,WAAW,CAAE;EACpD,MAAM,EAAE,MAAM,GAAG,MAAM,qBAAqB,SAAS;EACrD,MAAM,WAAW,MAAM,OAAO,KAAK,OAAO,KAAK;GAC7C,QAAQ,UAAU,YAAY,QAAQ,QAAQ,GAAG;GACjD;EACD,EAAC;EACF,MAAM,OACJ,aAAa,OAAO,MAAM,SAAS,aAAa,CAAC,IAChD,MAAM,SAAS,MAAM;AACxB,SAAO,KAAK,MAAM,eAAe;CAClC;;;;;;CAOD,MAAM,SAA8B;EAClC,MAAM,UAAU,EAAE,SAAS,KAAK,QAAS;AACzC,SAAO,qBAAqB,QAC1B,KAAK,SACL,KAAK,QACL,KAAK,SACL,KAAK,aACL,QACD;CACF;;;;;;;CAQD,MAAM,OAA4B;EAChC,MAAM,IAAI,MAAM,KAAK,QAAQ;EAC7B,MAAM,QAAQ,EAAE,QAAQ,CAAC,MAAM;EAC/B,MAAM,OAAO,EAAE,KAAK,SAAS,CAAC,MAAM;EACpC,MAAM,WAAW;GAAE,QAAQ,KAAK;GAAS;EAAO;AAChD,SAAO,CAAC,IAAIE,oCAAS;GAAE,aAAa;GAAM;EAAU,EAAE;CACvD;;;;;;CAOD,aAAa,UAEV;AACD,MAAI;GACF,MAAM,EAAE,MAAM,GAAG,MAAM,OAAO;AAC9B,UAAO,EAAE,KAAM;EAChB,SAAQ,GAAG;GACV,QAAQ,MAAM,EAAE;AAChB,SAAM,IAAI,MACR;EAEH;CACF;AACF"}