{"version":3,"file":"recursive_url.cjs","names":["VirtualConsole","BaseDocumentLoader","url: string","options: RecursiveUrlLoaderOptions","AsyncCaller","s: string","resource: string","options: { timeout: number } & RequestInit","html: string","baseUrl: string","JSDOM","standardizedLink: string","rawHtml: string","metadata: Record<string, any>","inputUrl: string","visited: Set<string>","childUrls: string[]"],"sources":["../../../src/document_loaders/web/recursive_url.ts"],"sourcesContent":["import { JSDOM, VirtualConsole } from \"jsdom\";\nimport { Document } from \"@langchain/core/documents\";\nimport { AsyncCaller } from \"@langchain/core/utils/async_caller\";\nimport {\n  BaseDocumentLoader,\n  DocumentLoader,\n} from \"@langchain/core/document_loaders/base\";\n\nconst virtualConsole = new VirtualConsole();\nvirtualConsole.on(\"error\", () => {});\n\nexport interface RecursiveUrlLoaderOptions {\n  excludeDirs?: string[];\n  extractor?: (text: string) => string;\n  maxDepth?: number;\n  timeout?: number;\n  preventOutside?: boolean;\n  callerOptions?: ConstructorParameters<typeof AsyncCaller>[0];\n}\n\nexport class RecursiveUrlLoader\n  extends BaseDocumentLoader\n  implements DocumentLoader\n{\n  private caller: AsyncCaller;\n\n  private url: string;\n\n  private excludeDirs: string[];\n\n  private extractor: (text: string) => string;\n\n  private maxDepth: number;\n\n  private timeout: number;\n\n  private preventOutside: boolean;\n\n  constructor(url: string, options: RecursiveUrlLoaderOptions) {\n    super();\n\n    this.caller = new AsyncCaller({\n      maxConcurrency: 64,\n      maxRetries: 0,\n      ...options.callerOptions,\n    });\n\n    this.url = url;\n    this.excludeDirs = options.excludeDirs ?? [];\n    this.extractor = options.extractor ?? ((s: string) => s);\n    this.maxDepth = options.maxDepth ?? 2;\n    this.timeout = options.timeout ?? 10000;\n    this.preventOutside = options.preventOutside ?? true;\n  }\n\n  private async fetchWithTimeout(\n    resource: string,\n    options: { timeout: number } & RequestInit\n  ): Promise<Response> {\n    const { timeout, ...rest } = options;\n    return this.caller.call(() =>\n      fetch(resource, { ...rest, signal: AbortSignal.timeout(timeout) })\n    );\n  }\n\n  private getChildLinks(html: string, baseUrl: string): Array<string> {\n    const allLinks = Array.from(\n      new JSDOM(html, { virtualConsole }).window.document.querySelectorAll(\"a\")\n    ).map((a) => a.href);\n    const absolutePaths = [];\n    const invalidPrefixes = [\"javascript:\", \"mailto:\", \"#\"];\n    const invalidSuffixes = [\n      \".css\",\n      \".js\",\n      \".ico\",\n      \".png\",\n      \".jpg\",\n      \".jpeg\",\n      \".gif\",\n      \".svg\",\n    ];\n\n    for (const link of allLinks) {\n      if (\n        invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||\n        invalidSuffixes.some((suffix) => link.endsWith(suffix))\n      )\n        continue;\n\n      let standardizedLink: string;\n\n      if (link.startsWith(\"http\")) {\n        standardizedLink = link;\n      } else if (link.startsWith(\"//\")) {\n        const base = new URL(baseUrl);\n        standardizedLink = base.protocol + link;\n      } else {\n        standardizedLink = new URL(link, baseUrl).href;\n      }\n\n      if (this.excludeDirs.some((exDir) => standardizedLink.startsWith(exDir)))\n        continue;\n\n      if (link.startsWith(\"http\")) {\n        const isAllowed = !this.preventOutside || link.startsWith(baseUrl);\n        if (isAllowed) absolutePaths.push(link);\n      } else if (link.startsWith(\"//\")) {\n        const base = new URL(baseUrl);\n        absolutePaths.push(base.protocol + link);\n      } else {\n        const newLink = new URL(link, baseUrl).href;\n        absolutePaths.push(newLink);\n      }\n    }\n\n    return Array.from(new Set(absolutePaths));\n  }\n\n  private extractMetadata(rawHtml: string, url: string) {\n    // eslint-disable-next-line @typescript-eslint/no-explicit-any\n    const metadata: Record<string, any> = { source: url };\n    const { document } = new JSDOM(rawHtml, { virtualConsole }).window;\n\n    const title = document.getElementsByTagName(\"title\")[0];\n    if (title) {\n      metadata.title = title.textContent;\n    }\n\n    const description = document.querySelector(\"meta[name=description]\");\n    if (description) {\n      metadata.description = description.getAttribute(\"content\");\n    }\n\n    const html = document.getElementsByTagName(\"html\")[0];\n    if (html) {\n      metadata.language = html.getAttribute(\"lang\");\n    }\n\n    return metadata;\n  }\n\n  private async getUrlAsDoc(url: string): Promise<Document | null> {\n    let res;\n    try {\n      res = await this.fetchWithTimeout(url, { timeout: this.timeout });\n      res = await res.text();\n    } catch {\n      return null;\n    }\n\n    return {\n      pageContent: this.extractor(res),\n      metadata: this.extractMetadata(res, url),\n    };\n  }\n\n  private async getChildUrlsRecursive(\n    inputUrl: string,\n    visited: Set<string> = new Set<string>(),\n    depth = 0\n  ): Promise<Document[]> {\n    if (depth >= this.maxDepth) return [];\n\n    let url = inputUrl;\n    if (!inputUrl.endsWith(\"/\")) url += \"/\";\n\n    const isExcluded = this.excludeDirs.some((exDir) => url.startsWith(exDir));\n    if (isExcluded) return [];\n\n    let res;\n    try {\n      res = await this.fetchWithTimeout(url, { timeout: this.timeout });\n      res = await res.text();\n    } catch {\n      return [];\n    }\n\n    const childUrls: string[] = this.getChildLinks(res, url);\n\n    const results = await Promise.all(\n      childUrls.map((childUrl) =>\n        (async () => {\n          if (visited.has(childUrl)) return null;\n          visited.add(childUrl);\n\n          const childDoc = await this.getUrlAsDoc(childUrl);\n          if (!childDoc) return null;\n\n          if (childUrl.endsWith(\"/\")) {\n            const childUrlResponses = await this.getChildUrlsRecursive(\n              childUrl,\n              visited,\n              depth + 1\n            );\n            return [childDoc, ...childUrlResponses];\n          }\n\n          return [childDoc];\n        })()\n      )\n    );\n\n    return results.flat().filter((docs) => docs !== null) as Document[];\n  }\n\n  async load(): Promise<Document[]> {\n    const rootDoc = await this.getUrlAsDoc(this.url);\n    if (!rootDoc) return [];\n\n    const docs = [rootDoc];\n    docs.push(\n      ...(await this.getChildUrlsRecursive(this.url, new Set([this.url])))\n    );\n    return docs;\n  }\n}\n"],"mappings":";;;;;;;;AAQA,MAAM,iBAAiB,IAAIA;AAC3B,eAAe,GAAG,SAAS,MAAM,CAAE,EAAC;AAWpC,IAAa,qBAAb,cACUC,0DAEV;CACE,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,AAAQ;CAER,YAAYC,KAAaC,SAAoC;EAC3D,OAAO;EAEP,KAAK,SAAS,IAAIC,gDAAY;GAC5B,gBAAgB;GAChB,YAAY;GACZ,GAAG,QAAQ;EACZ;EAED,KAAK,MAAM;EACX,KAAK,cAAc,QAAQ,eAAe,CAAE;EAC5C,KAAK,YAAY,QAAQ,cAAc,CAACC,MAAc;EACtD,KAAK,WAAW,QAAQ,YAAY;EACpC,KAAK,UAAU,QAAQ,WAAW;EAClC,KAAK,iBAAiB,QAAQ,kBAAkB;CACjD;CAED,MAAc,iBACZC,UACAC,SACmB;EACnB,MAAM,EAAE,QAAS,GAAG,MAAM,GAAG;AAC7B,SAAO,KAAK,OAAO,KAAK,MACtB,MAAM,UAAU;GAAE,GAAG;GAAM,QAAQ,YAAY,QAAQ,QAAQ;EAAE,EAAC,CACnE;CACF;CAED,AAAQ,cAAcC,MAAcC,SAAgC;EAClE,MAAM,WAAW,MAAM,KACrB,IAAIC,YAAM,MAAM,EAAE,eAAgB,GAAE,OAAO,SAAS,iBAAiB,IAAI,CAC1E,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK;EACpB,MAAM,gBAAgB,CAAE;EACxB,MAAM,kBAAkB;GAAC;GAAe;GAAW;EAAI;EACvD,MAAM,kBAAkB;GACtB;GACA;GACA;GACA;GACA;GACA;GACA;GACA;EACD;AAED,OAAK,MAAM,QAAQ,UAAU;AAC3B,OACE,gBAAgB,KAAK,CAAC,WAAW,KAAK,WAAW,OAAO,CAAC,IACzD,gBAAgB,KAAK,CAAC,WAAW,KAAK,SAAS,OAAO,CAAC,CAEvD;GAEF,IAAIC;AAEJ,OAAI,KAAK,WAAW,OAAO,EACzB,mBAAmB;YACV,KAAK,WAAW,KAAK,EAAE;IAChC,MAAM,OAAO,IAAI,IAAI;IACrB,mBAAmB,KAAK,WAAW;GACpC,OACC,mBAAmB,IAAI,IAAI,MAAM,SAAS;AAG5C,OAAI,KAAK,YAAY,KAAK,CAAC,UAAU,iBAAiB,WAAW,MAAM,CAAC,CACtE;AAEF,OAAI,KAAK,WAAW,OAAO,EAAE;IAC3B,MAAM,YAAY,CAAC,KAAK,kBAAkB,KAAK,WAAW,QAAQ;AAClE,QAAI,WAAW,cAAc,KAAK,KAAK;GACxC,WAAU,KAAK,WAAW,KAAK,EAAE;IAChC,MAAM,OAAO,IAAI,IAAI;IACrB,cAAc,KAAK,KAAK,WAAW,KAAK;GACzC,OAAM;IACL,MAAM,UAAU,IAAI,IAAI,MAAM,SAAS;IACvC,cAAc,KAAK,QAAQ;GAC5B;EACF;AAED,SAAO,MAAM,KAAK,IAAI,IAAI,eAAe;CAC1C;CAED,AAAQ,gBAAgBC,SAAiBV,KAAa;EAEpD,MAAMW,WAAgC,EAAE,QAAQ,IAAK;EACrD,MAAM,EAAE,UAAU,GAAG,IAAIH,YAAM,SAAS,EAAE,eAAgB,GAAE;EAE5D,MAAM,QAAQ,SAAS,qBAAqB,QAAQ,CAAC;AACrD,MAAI,OACF,SAAS,QAAQ,MAAM;EAGzB,MAAM,cAAc,SAAS,cAAc,yBAAyB;AACpE,MAAI,aACF,SAAS,cAAc,YAAY,aAAa,UAAU;EAG5D,MAAM,OAAO,SAAS,qBAAqB,OAAO,CAAC;AACnD,MAAI,MACF,SAAS,WAAW,KAAK,aAAa,OAAO;AAG/C,SAAO;CACR;CAED,MAAc,YAAYR,KAAuC;EAC/D,IAAI;AACJ,MAAI;GACF,MAAM,MAAM,KAAK,iBAAiB,KAAK,EAAE,SAAS,KAAK,QAAS,EAAC;GACjE,MAAM,MAAM,IAAI,MAAM;EACvB,QAAO;AACN,UAAO;EACR;AAED,SAAO;GACL,aAAa,KAAK,UAAU,IAAI;GAChC,UAAU,KAAK,gBAAgB,KAAK,IAAI;EACzC;CACF;CAED,MAAc,sBACZY,UACAC,0BAAuB,IAAI,OAC3B,QAAQ,GACa;AACrB,MAAI,SAAS,KAAK,SAAU,QAAO,CAAE;EAErC,IAAI,MAAM;AACV,MAAI,CAAC,SAAS,SAAS,IAAI,EAAE,OAAO;EAEpC,MAAM,aAAa,KAAK,YAAY,KAAK,CAAC,UAAU,IAAI,WAAW,MAAM,CAAC;AAC1E,MAAI,WAAY,QAAO,CAAE;EAEzB,IAAI;AACJ,MAAI;GACF,MAAM,MAAM,KAAK,iBAAiB,KAAK,EAAE,SAAS,KAAK,QAAS,EAAC;GACjE,MAAM,MAAM,IAAI,MAAM;EACvB,QAAO;AACN,UAAO,CAAE;EACV;EAED,MAAMC,YAAsB,KAAK,cAAc,KAAK,IAAI;EAExD,MAAM,UAAU,MAAM,QAAQ,IAC5B,UAAU,IAAI,CAAC,cACZ,YAAY;AACX,OAAI,QAAQ,IAAI,SAAS,CAAE,QAAO;GAClC,QAAQ,IAAI,SAAS;GAErB,MAAM,WAAW,MAAM,KAAK,YAAY,SAAS;AACjD,OAAI,CAAC,SAAU,QAAO;AAEtB,OAAI,SAAS,SAAS,IAAI,EAAE;IAC1B,MAAM,oBAAoB,MAAM,KAAK,sBACnC,UACA,SACA,QAAQ,EACT;AACD,WAAO,CAAC,UAAU,GAAG,iBAAkB;GACxC;AAED,UAAO,CAAC,QAAS;EAClB,IAAG,CACL,CACF;AAED,SAAO,QAAQ,MAAM,CAAC,OAAO,CAAC,SAAS,SAAS,KAAK;CACtD;CAED,MAAM,OAA4B;EAChC,MAAM,UAAU,MAAM,KAAK,YAAY,KAAK,IAAI;AAChD,MAAI,CAAC,QAAS,QAAO,CAAE;EAEvB,MAAM,OAAO,CAAC,OAAQ;EACtB,KAAK,KACH,GAAI,MAAM,KAAK,sBAAsB,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,GAAI,GAAE,CACpE;AACD,SAAO;CACR;AACF"}