'use strict'; var Bottleneck = require('bottleneck'); var parseLinkHeader = require('@web3-storage/parse-link-header'); var createHttpError = require('http-errors'); const ONE_YEAR = 1e3 * 60 * 60 * 24 * 365; function getResponseDate(res) { const val = res.headers.get("Date"); return val ? Date.parse(val) : Date.now(); } function getResetHeader(response, name = "Retry-After", format = "seconds") { const value = response.headers.get(name); if (value) { let parsed; switch (format) { case "datetime": parsed = Date.parse(value); break; case "seconds": parsed = parseInt(value, 10) * 1e3; break; case "milliseconds": parsed = parseInt(value, 10); break; } if (isNaN(parsed) === false) { if (parsed > ONE_YEAR) { return parsed - getResponseDate(response); } return parsed; } else { throw new Error(`Could not coerce value "${value}" to a number`); } } } function joinBaseUrl(path, baseUrl) { if (path.startsWith("http") || typeof baseUrl !== "string") { return path; } return [baseUrl, path].map((part) => part.replace(/^\/|\/$/, "")).join("/"); } const backoff = (retries) => Math.pow(retries + 1, 2) * 1e3; function handleFailed(options, error, info) { const retry = info.retryCount < options.maxRetries; if (retry) { if (createHttpError.isHttpError(error) && error?.response instanceof Response) { const response = error.response; if (response.status === 429) { const wait = getResetHeader(response, options.rateLimitHeader, options.resetFormat); if (wait) { return wait + 1e3; } } if (!options.doNotRetry?.includes(response.status)) { return backoff(info.retryCount); } } if (error.name === "TimeoutError") { return backoff(info.retryCount); } } } async function makeRequest(url, init = {}, timeout = 3e4) { const signal = AbortSignal.timeout(timeout); const request = new Request(url, { ...init, signal }); const response = await fetch(request); if (response.ok) { return response; } throw createHttpError(response.status, { request, response }); } const defaults = { // Throttle options maxConcurrency: 10, minRequestTime: 0, // Retry options maxRetries: 3, doNotRetry: [400, 401, 403, 404, 422, 451], // Rate limit options rateLimitHeader: "Retry-After", resetFormat: "seconds" }; class HardenedFetch { constructor(options = {}) { this.options = Object.assign({}, defaults, options); this.queue = new Bottleneck({ maxConcurrent: this.options.maxConcurrency, minTime: this.options.minRequestTime }); this.queue.on("failed", handleFailed.bind(null, this.options)); } fetch(url, init = {}, timeout = 3e4) { const resolvedUrl = joinBaseUrl(url, this.options.baseUrl); if (this.options.defaultHeaders) { const headers = Object.assign({}, this.options.defaultHeaders, init.headers); init = Object.assign({}, init, { headers }); } return this.queue.schedule(makeRequest, resolvedUrl, init, timeout); } async *paginatedFetch(url, init = {}, timeout = 3e4) { let nextUrl = url; while (nextUrl) { const response = await this.fetch(nextUrl, init, timeout); const linkHeader = response.headers.get("Link"); const links = parseLinkHeader.parseLinkHeader(linkHeader); nextUrl = links?.next ? links.next.url : null; yield { response, done: !nextUrl }; } } } exports.HardenedFetch = HardenedFetch;