{"version":3,"file":"arxiv.cjs","names":["query: string","arxivIds: string","XMLParser","pdfUrl: string","PDFLoader","docs: Document[]","results: ArxivEntry[]","Document","entry: any","authors: string[]","author: any","links: any[]","link: any"],"sources":["../../src/utils/arxiv.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\nimport { Document } from \"@langchain/core/documents\";\nimport { XMLParser } from \"fast-xml-parser\";\n\nimport { PDFLoader } from \"../document_loaders/fs/pdf.js\";\n\n// Interface for processed arXiv entry\ninterface ArxivEntry {\n  id: string;\n  title: string;\n  summary: string;\n  published: string;\n  updated: string;\n  authors: string[];\n  pdfUrl: string;\n  links: any[];\n}\n\n// Used to check if the query is an arXiv ID, or a natural language query\nexport function isArXivIdentifier(query: string): boolean {\n  const arxivIdRegex = /^\\d{4}\\.\\d{4,5}(v\\d+)?$|^\\d{7}(\\.\\d+)?(v\\d+)?$/;\n  return arxivIdRegex.test(query.trim());\n}\n\n// Used to fetch direct arXiv articles by IDs (supports multiple IDs)\nexport async function fetchDirectArxivArticle(\n  arxivIds: string\n): Promise<ArxivEntry[]> {\n  try {\n    const idList = arxivIds\n      .split(/[\\s,]+/)\n      .map((id) => id.trim())\n      .filter(Boolean)\n      .join(\",\");\n    const url = `http://export.arxiv.org/api/query?id_list=${idList}`;\n    const response = await fetch(url);\n\n    if (!response.ok) {\n      throw new Error(`HTTP error! status: ${response.status}`);\n    }\n\n    const xml = await response.text();\n\n    const parser = new XMLParser({\n      ignoreAttributes: false,\n      attributeNamePrefix: \"@_\",\n    });\n    const result = parser.parse(xml);\n    let entries = result.feed.entry;\n\n    if (!entries) {\n      return [];\n    }\n\n    // Ensure entries is an array\n    if (!Array.isArray(entries)) {\n      entries = [entries];\n    }\n\n    const processedEntries = entries.map(processEntry);\n\n    return processedEntries;\n  } catch {\n    throw new Error(`Failed to fetch articles with IDs ${arxivIds}`);\n  }\n}\n\n// Used to fetch arXiv results by natural language query with maxResults parameter\nexport async function fetchArxivResultsByQuery(\n  query: string,\n  start = 0,\n  maxResults = 10\n): Promise<ArxivEntry[]> {\n  try {\n    const encodedQuery = encodeURIComponent(query);\n    const url = `http://export.arxiv.org/api/query?search_query=all:${encodedQuery}&start=${start}&max_results=${maxResults}`;\n    const response = await fetch(url);\n\n    if (!response.ok) {\n      throw new Error(`HTTP error! status: ${response.status}`);\n    }\n\n    const xml = await response.text();\n\n    const parser = new XMLParser({\n      ignoreAttributes: false,\n      attributeNamePrefix: \"@_\",\n    });\n    const result = parser.parse(xml);\n    let entries = result.feed.entry;\n\n    if (!entries) {\n      return [];\n    }\n\n    // Ensure entries is an array\n    if (!Array.isArray(entries)) {\n      entries = [entries];\n    }\n\n    const processedEntries = entries.map(processEntry);\n\n    return processedEntries;\n  } catch {\n    throw new Error(`Failed to fetch articles with query \"${query}\"`);\n  }\n}\n\n// Used to search for arXiv articles with a maxResults parameter\nexport async function searchArxiv(\n  query: string,\n  maxResults = 3\n): Promise<ArxivEntry[]> {\n  if (isArXivIdentifier(query)) {\n    return await fetchDirectArxivArticle(query);\n  } else {\n    return await fetchArxivResultsByQuery(query, 0, maxResults);\n  }\n}\n\n// Used to fetch and parse PDF to text\nexport async function fetchAndParsePDF(pdfUrl: string): Promise<string> {\n  try {\n    // Fetch the PDF\n    const response = await fetch(pdfUrl);\n\n    if (!response.ok) {\n      throw new Error(`HTTP error! status: ${response.status}`);\n    }\n\n    const buffer = await response.arrayBuffer();\n\n    // Convert the ArrayBuffer to a Blob\n    const blob = new Blob([buffer], { type: \"application/pdf\" });\n\n    // Use PDFLoader to process the PDF\n    const loader = new PDFLoader(blob, { splitPages: false }); // Pass the Blob\n    const docs: Document[] = await loader.load();\n\n    // Combine all document content into a single string\n    const content = docs.map((doc) => doc.pageContent).join(\"\\n\\n\");\n    return content;\n  } catch {\n    throw new Error(`Failed to fetch or parse PDF from ${pdfUrl}`);\n  }\n}\n\n// Used to load raw text from each search result, and convert to Document instances\nexport async function loadDocsFromResults(\n  results: ArxivEntry[]\n): Promise<Document[]> {\n  const docs: Document[] = [];\n  for (const result of results) {\n    const pdfUrl = result.pdfUrl;\n    try {\n      const pdfContent = await fetchAndParsePDF(pdfUrl);\n      const metadata = {\n        id: result.id,\n        title: result.title,\n        authors: result.authors,\n        published: result.published,\n        updated: result.updated,\n        source: \"arxiv\",\n        url: result.id,\n        summary: result.summary,\n      };\n      const doc = new Document({\n        pageContent: pdfContent,\n        metadata,\n      });\n      docs.push(doc);\n    } catch {\n      throw new Error(`Error loading document from ${pdfUrl}`);\n    }\n  }\n  return docs;\n}\n\n// Used to convert metadata and summaries to Document instances\nexport function getDocsFromSummaries(results: ArxivEntry[]): Document[] {\n  const docs: Document[] = [];\n  for (const result of results) {\n    const metadata = {\n      id: result.id,\n      title: result.title,\n      authors: result.authors,\n      published: result.published,\n      updated: result.updated,\n      source: \"arxiv\",\n      url: result.id,\n    };\n    const doc = new Document({\n      pageContent: result.summary,\n      metadata,\n    });\n    docs.push(doc);\n  }\n  return docs;\n}\n\n// Helper function to process each arXiv entry\nfunction processEntry(entry: any): ArxivEntry {\n  const id = entry.id;\n  const title = entry.title.replace(/\\s+/g, \" \").trim();\n  const summary = entry.summary.replace(/\\s+/g, \" \").trim();\n  const published = entry.published;\n  const updated = entry.updated;\n\n  // Extract authors\n  let authors: string[] = [];\n  if (Array.isArray(entry.author)) {\n    authors = entry.author.map((author: any) => author.name);\n  } else if (entry.author) {\n    authors = [entry.author.name];\n  }\n\n  // Extract links\n  let links: any[] = [];\n  if (Array.isArray(entry.link)) {\n    links = entry.link;\n  } else if (entry.link) {\n    links = [entry.link];\n  }\n\n  // Extract PDF link\n  let pdfUrl = `${id.replace(\"/abs/\", \"/pdf/\")}.pdf`;\n  const pdfLinkObj = links.find((link: any) => link[\"@_title\"] === \"pdf\");\n  if (pdfLinkObj && pdfLinkObj[\"@_href\"]) {\n    pdfUrl = pdfLinkObj[\"@_href\"];\n  }\n\n  return {\n    id,\n    title,\n    summary,\n    published,\n    updated,\n    authors,\n    pdfUrl,\n    links,\n  };\n}\n"],"mappings":";;;;;;AAmBA,SAAgB,kBAAkBA,OAAwB;CACxD,MAAM,eAAe;AACrB,QAAO,aAAa,KAAK,MAAM,MAAM,CAAC;AACvC;AAGD,eAAsB,wBACpBC,UACuB;AACvB,KAAI;EACF,MAAM,SAAS,SACZ,MAAM,SAAS,CACf,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,CACtB,OAAO,QAAQ,CACf,KAAK,IAAI;EACZ,MAAM,MAAM,CAAC,0CAA0C,EAAE,QAAQ;EACjE,MAAM,WAAW,MAAM,MAAM,IAAI;AAEjC,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM,CAAC,oBAAoB,EAAE,SAAS,QAAQ;EAG1D,MAAM,MAAM,MAAM,SAAS,MAAM;EAEjC,MAAM,SAAS,IAAIC,0BAAU;GAC3B,kBAAkB;GAClB,qBAAqB;EACtB;EACD,MAAM,SAAS,OAAO,MAAM,IAAI;EAChC,IAAI,UAAU,OAAO,KAAK;AAE1B,MAAI,CAAC,QACH,QAAO,CAAE;AAIX,MAAI,CAAC,MAAM,QAAQ,QAAQ,EACzB,UAAU,CAAC,OAAQ;EAGrB,MAAM,mBAAmB,QAAQ,IAAI,aAAa;AAElD,SAAO;CACR,QAAO;AACN,QAAM,IAAI,MAAM,CAAC,kCAAkC,EAAE,UAAU;CAChE;AACF;AAGD,eAAsB,yBACpBF,OACA,QAAQ,GACR,aAAa,IACU;AACvB,KAAI;EACF,MAAM,eAAe,mBAAmB,MAAM;EAC9C,MAAM,MAAM,CAAC,mDAAmD,EAAE,aAAa,OAAO,EAAE,MAAM,aAAa,EAAE,YAAY;EACzH,MAAM,WAAW,MAAM,MAAM,IAAI;AAEjC,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM,CAAC,oBAAoB,EAAE,SAAS,QAAQ;EAG1D,MAAM,MAAM,MAAM,SAAS,MAAM;EAEjC,MAAM,SAAS,IAAIE,0BAAU;GAC3B,kBAAkB;GAClB,qBAAqB;EACtB;EACD,MAAM,SAAS,OAAO,MAAM,IAAI;EAChC,IAAI,UAAU,OAAO,KAAK;AAE1B,MAAI,CAAC,QACH,QAAO,CAAE;AAIX,MAAI,CAAC,MAAM,QAAQ,QAAQ,EACzB,UAAU,CAAC,OAAQ;EAGrB,MAAM,mBAAmB,QAAQ,IAAI,aAAa;AAElD,SAAO;CACR,QAAO;AACN,QAAM,IAAI,MAAM,CAAC,qCAAqC,EAAE,MAAM,CAAC,CAAC;CACjE;AACF;AAGD,eAAsB,YACpBF,OACA,aAAa,GACU;AACvB,KAAI,kBAAkB,MAAM,CAC1B,QAAO,MAAM,wBAAwB,MAAM;KAE3C,QAAO,MAAM,yBAAyB,OAAO,GAAG,WAAW;AAE9D;AAGD,eAAsB,iBAAiBG,QAAiC;AACtE,KAAI;EAEF,MAAM,WAAW,MAAM,MAAM,OAAO;AAEpC,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM,CAAC,oBAAoB,EAAE,SAAS,QAAQ;EAG1D,MAAM,SAAS,MAAM,SAAS,aAAa;EAG3C,MAAM,OAAO,IAAI,KAAK,CAAC,MAAO,GAAE,EAAE,MAAM,kBAAmB;EAG3D,MAAM,SAAS,IAAIC,0CAAU,MAAM,EAAE,YAAY,MAAO;EACxD,MAAMC,OAAmB,MAAM,OAAO,MAAM;EAG5C,MAAM,UAAU,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,KAAK,OAAO;AAC/D,SAAO;CACR,QAAO;AACN,QAAM,IAAI,MAAM,CAAC,kCAAkC,EAAE,QAAQ;CAC9D;AACF;AAGD,eAAsB,oBACpBC,SACqB;CACrB,MAAMD,OAAmB,CAAE;AAC3B,MAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,SAAS,OAAO;AACtB,MAAI;GACF,MAAM,aAAa,MAAM,iBAAiB,OAAO;GACjD,MAAM,WAAW;IACf,IAAI,OAAO;IACX,OAAO,OAAO;IACd,SAAS,OAAO;IAChB,WAAW,OAAO;IAClB,SAAS,OAAO;IAChB,QAAQ;IACR,KAAK,OAAO;IACZ,SAAS,OAAO;GACjB;GACD,MAAM,MAAM,IAAIE,oCAAS;IACvB,aAAa;IACb;GACD;GACD,KAAK,KAAK,IAAI;EACf,QAAO;AACN,SAAM,IAAI,MAAM,CAAC,4BAA4B,EAAE,QAAQ;EACxD;CACF;AACD,QAAO;AACR;AAGD,SAAgB,qBAAqBD,SAAmC;CACtE,MAAMD,OAAmB,CAAE;AAC3B,MAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,WAAW;GACf,IAAI,OAAO;GACX,OAAO,OAAO;GACd,SAAS,OAAO;GAChB,WAAW,OAAO;GAClB,SAAS,OAAO;GAChB,QAAQ;GACR,KAAK,OAAO;EACb;EACD,MAAM,MAAM,IAAIE,oCAAS;GACvB,aAAa,OAAO;GACpB;EACD;EACD,KAAK,KAAK,IAAI;CACf;AACD,QAAO;AACR;AAGD,SAAS,aAAaC,OAAwB;CAC5C,MAAM,KAAK,MAAM;CACjB,MAAM,QAAQ,MAAM,MAAM,QAAQ,QAAQ,IAAI,CAAC,MAAM;CACrD,MAAM,UAAU,MAAM,QAAQ,QAAQ,QAAQ,IAAI,CAAC,MAAM;CACzD,MAAM,YAAY,MAAM;CACxB,MAAM,UAAU,MAAM;CAGtB,IAAIC,UAAoB,CAAE;AAC1B,KAAI,MAAM,QAAQ,MAAM,OAAO,EAC7B,UAAU,MAAM,OAAO,IAAI,CAACC,WAAgB,OAAO,KAAK;UAC/C,MAAM,QACf,UAAU,CAAC,MAAM,OAAO,IAAK;CAI/B,IAAIC,QAAe,CAAE;AACrB,KAAI,MAAM,QAAQ,MAAM,KAAK,EAC3B,QAAQ,MAAM;UACL,MAAM,MACf,QAAQ,CAAC,MAAM,IAAK;CAItB,IAAI,SAAS,GAAG,GAAG,QAAQ,SAAS,QAAQ,CAAC,IAAI,CAAC;CAClD,MAAM,aAAa,MAAM,KAAK,CAACC,SAAc,KAAK,eAAe,MAAM;AACvE,KAAI,cAAc,WAAW,WAC3B,SAAS,WAAW;AAGtB,QAAO;EACL;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACD;AACF"}