{"version":3,"file":"sitemap.cjs","names":["CheerioWebBaseLoader","webPath: string","params: SitemapLoaderParams","url: string","elements: Array","documents: Array","Document","documents: DocumentInterface[]"],"sources":["../../../src/document_loaders/web/sitemap.ts"],"sourcesContent":["import { Document, DocumentInterface } from \"@langchain/core/documents\";\nimport { chunkArray } from \"@langchain/core/utils/chunk_array\";\nimport { CheerioWebBaseLoader, CheerioWebBaseLoaderParams } from \"./cheerio.js\";\n\n/**\n * Interface representing the parameters for initializing a SitemapLoader.\n * @interface SitemapLoaderParams\n * @extends CheerioWebBaseLoaderParams\n */\nexport interface SitemapLoaderParams extends CheerioWebBaseLoaderParams {\n /**\n * @property {(string | RegExp)[] | undefined} filterUrls - A list of regexes. Only URLs that match one of the filter URLs will be loaded.\n * WARNING: The filter URLs are interpreted as regular expressions. Escape special characters if needed.\n */\n filterUrls?: (string | RegExp)[];\n /**\n * The size to chunk the sitemap URLs into for scraping.\n * @default {300}\n */\n chunkSize?: number;\n}\n\nconst DEFAULT_CHUNK_SIZE = 300;\n\ntype SiteMapElement = {\n loc: string;\n changefreq?: string;\n lastmod?: string;\n priority?: string;\n};\n\nexport class SitemapLoader\n extends CheerioWebBaseLoader\n implements SitemapLoaderParams\n{\n allowUrlPatterns: (string | RegExp)[] | undefined;\n\n chunkSize: number;\n\n constructor(\n public webPath: string,\n params: SitemapLoaderParams = {}\n ) {\n const paramsWithDefaults = { chunkSize: DEFAULT_CHUNK_SIZE, ...params };\n let path = webPath.endsWith(\"/\") ? webPath.slice(0, -1) : webPath;\n // Allow for custom sitemap paths to be passed in with the url.\n path = path.endsWith(\".xml\") ? path : `${path}/sitemap.xml`;\n super(path, paramsWithDefaults);\n\n this.webPath = path;\n this.allowUrlPatterns = paramsWithDefaults.filterUrls;\n this.chunkSize = paramsWithDefaults.chunkSize;\n }\n\n _checkUrlPatterns(url: string): boolean {\n if (!this.allowUrlPatterns) {\n return false;\n }\n return !this.allowUrlPatterns.some(\n (pattern) => !new RegExp(pattern).test(url)\n );\n }\n\n async parseSitemap() {\n const $ = await CheerioWebBaseLoader._scrape(\n this.webPath,\n this.caller,\n this.timeout,\n this.textDecoder,\n {\n xmlMode: true,\n xml: true,\n }\n );\n\n const elements: Array = [];\n\n $(\"url\").each((_, element) => {\n const loc = $(element).find(\"loc\").text();\n if (!loc) {\n return;\n }\n\n if (this._checkUrlPatterns(loc)) {\n return;\n }\n\n const changefreq = $(element).find(\"changefreq\").text();\n const lastmod = $(element).find(\"lastmod\").text();\n const priority = $(element).find(\"priority\").text();\n\n elements.push({ loc, changefreq, lastmod, priority });\n });\n\n $(\"sitemap\").each((_, element) => {\n const loc = $(element).find(\"loc\").text();\n if (!loc) {\n return;\n }\n const changefreq = $(element).find(\"changefreq\").text();\n const lastmod = $(element).find(\"lastmod\").text();\n const priority = $(element).find(\"priority\").text();\n\n elements.push({ loc, changefreq, lastmod, priority });\n });\n\n return elements;\n }\n\n async _loadSitemapUrls(\n elements: Array\n ): Promise {\n const all = await CheerioWebBaseLoader.scrapeAll(\n elements.map((ele) => ele.loc),\n this.caller,\n this.timeout,\n this.textDecoder\n );\n const documents: Array = all.map(($, i) => {\n if (!elements[i]) {\n throw new Error(\"Scraped docs and elements not in sync\");\n }\n const text = $(this.selector).text();\n const { loc: source, ...metadata } = elements[i];\n\n // extract page metadata\n const description = $(\"meta[name='description']\").attr(\"content\");\n const title = $(\"meta[property='og:title']\").attr(\"content\");\n const lang = $(\"meta[property='og:locale']\").attr(\"content\");\n\n return new Document({\n pageContent: text,\n metadata: {\n ...metadata,\n description,\n title,\n lang,\n source: source.trim(),\n },\n });\n });\n return documents;\n }\n\n async load(): Promise {\n const elements = await this.parseSitemap();\n const chunks = chunkArray(elements, this.chunkSize);\n\n const documents: DocumentInterface[] = [];\n for await (const chunk of chunks) {\n const chunkedDocuments = await this._loadSitemapUrls(chunk);\n documents.push(...chunkedDocuments);\n }\n return documents;\n }\n}\n"],"mappings":";;;;;;;;AAsBA,MAAM,qBAAqB;AAS3B,IAAa,gBAAb,cACUA,0DAEV;CACE;CAEA;CAEA,YACSC,SACPC,SAA8B,CAAE,GAChC;EACA,MAAM,qBAAqB;GAAE,WAAW;GAAoB,GAAG;EAAQ;EACvE,IAAI,OAAO,QAAQ,SAAS,IAAI,GAAG,QAAQ,MAAM,GAAG,GAAG,GAAG;EAE1D,OAAO,KAAK,SAAS,OAAO,GAAG,OAAO,GAAG,KAAK,YAAY,CAAC;EAC3D,MAAM,MAAM,mBAAmB;EAPxB;EASP,KAAK,UAAU;EACf,KAAK,mBAAmB,mBAAmB;EAC3C,KAAK,YAAY,mBAAmB;CACrC;CAED,kBAAkBC,KAAsB;AACtC,MAAI,CAAC,KAAK,iBACR,QAAO;AAET,SAAO,CAAC,KAAK,iBAAiB,KAC5B,CAAC,YAAY,CAAC,IAAI,OAAO,SAAS,KAAK,IAAI,CAC5C;CACF;CAED,MAAM,eAAe;EACnB,MAAM,IAAI,MAAMH,0DAAqB,QACnC,KAAK,SACL,KAAK,QACL,KAAK,SACL,KAAK,aACL;GACE,SAAS;GACT,KAAK;EACN,EACF;EAED,MAAMI,WAAkC,CAAE;EAE1C,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,YAAY;GAC5B,MAAM,MAAM,EAAE,QAAQ,CAAC,KAAK,MAAM,CAAC,MAAM;AACzC,OAAI,CAAC,IACH;AAGF,OAAI,KAAK,kBAAkB,IAAI,CAC7B;GAGF,MAAM,aAAa,EAAE,QAAQ,CAAC,KAAK,aAAa,CAAC,MAAM;GACvD,MAAM,UAAU,EAAE,QAAQ,CAAC,KAAK,UAAU,CAAC,MAAM;GACjD,MAAM,WAAW,EAAE,QAAQ,CAAC,KAAK,WAAW,CAAC,MAAM;GAEnD,SAAS,KAAK;IAAE;IAAK;IAAY;IAAS;GAAU,EAAC;EACtD,EAAC;EAEF,EAAE,UAAU,CAAC,KAAK,CAAC,GAAG,YAAY;GAChC,MAAM,MAAM,EAAE,QAAQ,CAAC,KAAK,MAAM,CAAC,MAAM;AACzC,OAAI,CAAC,IACH;GAEF,MAAM,aAAa,EAAE,QAAQ,CAAC,KAAK,aAAa,CAAC,MAAM;GACvD,MAAM,UAAU,EAAE,QAAQ,CAAC,KAAK,UAAU,CAAC,MAAM;GACjD,MAAM,WAAW,EAAE,QAAQ,CAAC,KAAK,WAAW,CAAC,MAAM;GAEnD,SAAS,KAAK;IAAE;IAAK;IAAY;IAAS;GAAU,EAAC;EACtD,EAAC;AAEF,SAAO;CACR;CAED,MAAM,iBACJA,UAC8B;EAC9B,MAAM,MAAM,MAAMJ,0DAAqB,UACrC,SAAS,IAAI,CAAC,QAAQ,IAAI,IAAI,EAC9B,KAAK,QACL,KAAK,SACL,KAAK,YACN;EACD,MAAMK,YAAsC,IAAI,IAAI,CAAC,GAAG,MAAM;AAC5D,OAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM;GAElB,MAAM,OAAO,EAAE,KAAK,SAAS,CAAC,MAAM;GACpC,MAAM,EAAE,KAAK,OAAQ,GAAG,UAAU,GAAG,SAAS;GAG9C,MAAM,cAAc,EAAE,2BAA2B,CAAC,KAAK,UAAU;GACjE,MAAM,QAAQ,EAAE,4BAA4B,CAAC,KAAK,UAAU;GAC5D,MAAM,OAAO,EAAE,6BAA6B,CAAC,KAAK,UAAU;AAE5D,UAAO,IAAIC,oCAAS;IAClB,aAAa;IACb,UAAU;KACR,GAAG;KACH;KACA;KACA;KACA,QAAQ,OAAO,MAAM;IACtB;GACF;EACF,EAAC;AACF,SAAO;CACR;CAED,MAAM,OAA4B;EAChC,MAAM,WAAW,MAAM,KAAK,cAAc;EAC1C,MAAM,4DAAoB,UAAU,KAAK,UAAU;EAEnD,MAAMC,YAAiC,CAAE;AACzC,aAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,mBAAmB,MAAM,KAAK,iBAAiB,MAAM;GAC3D,UAAU,KAAK,GAAG,iBAAiB;EACpC;AACD,SAAO;CACR;AACF"}