1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 | import http from 'node:http';
|
10 | import https from 'node:https';
|
11 | import zlib from 'node:zlib';
|
12 | import Stream, {PassThrough, pipeline as pump} from 'node:stream';
|
13 | import {Buffer} from 'node:buffer';
|
14 |
|
15 | import dataUriToBuffer from 'data-uri-to-buffer';
|
16 |
|
17 | import {writeToStream, clone} from './body.js';
|
18 | import Response from './response.js';
|
19 | import Headers, {fromRawHeaders} from './headers.js';
|
20 | import Request, {getNodeRequestOptions} from './request.js';
|
21 | import {FetchError} from './errors/fetch-error.js';
|
22 | import {AbortError} from './errors/abort-error.js';
|
23 | import {isRedirect} from './utils/is-redirect.js';
|
24 | import {FormData} from 'formdata-polyfill/esm.min.js';
|
25 | import {isDomainOrSubdomain, isSameProtocol} from './utils/is.js';
|
26 | import {parseReferrerPolicyFromHeader} from './utils/referrer.js';
|
27 | import {
|
28 | Blob,
|
29 | File,
|
30 | fileFromSync,
|
31 | fileFrom,
|
32 | blobFromSync,
|
33 | blobFrom
|
34 | } from 'fetch-blob/from.js';
|
35 |
|
36 | export {FormData, Headers, Request, Response, FetchError, AbortError, isRedirect};
|
37 | export {Blob, File, fileFromSync, fileFrom, blobFromSync, blobFrom};
|
38 |
|
39 | const supportedSchemas = new Set(['data:', 'http:', 'https:']);
|
40 |
|
41 |
|
42 |
|
43 |
|
44 |
|
45 |
|
46 |
|
47 |
|
48 | export default async function fetch(url, options_) {
|
49 | return new Promise((resolve, reject) => {
|
50 |
|
51 | const request = new Request(url, options_);
|
52 | const {parsedURL, options} = getNodeRequestOptions(request);
|
53 | if (!supportedSchemas.has(parsedURL.protocol)) {
|
54 | throw new TypeError(`node-fetch cannot load ${url}. URL scheme "${parsedURL.protocol.replace(/:$/, '')}" is not supported.`);
|
55 | }
|
56 |
|
57 | if (parsedURL.protocol === 'data:') {
|
58 | const data = dataUriToBuffer(request.url);
|
59 | const response = new Response(data, {headers: {'Content-Type': data.typeFull}});
|
60 | resolve(response);
|
61 | return;
|
62 | }
|
63 |
|
64 |
|
65 | const send = (parsedURL.protocol === 'https:' ? https : http).request;
|
66 | const {signal} = request;
|
67 | let response = null;
|
68 |
|
69 | const abort = () => {
|
70 | const error = new AbortError('The operation was aborted.');
|
71 | reject(error);
|
72 | if (request.body && request.body instanceof Stream.Readable) {
|
73 | request.body.destroy(error);
|
74 | }
|
75 |
|
76 | if (!response || !response.body) {
|
77 | return;
|
78 | }
|
79 |
|
80 | response.body.emit('error', error);
|
81 | };
|
82 |
|
83 | if (signal && signal.aborted) {
|
84 | abort();
|
85 | return;
|
86 | }
|
87 |
|
88 | const abortAndFinalize = () => {
|
89 | abort();
|
90 | finalize();
|
91 | };
|
92 |
|
93 |
|
94 | const request_ = send(parsedURL.toString(), options);
|
95 |
|
96 | if (signal) {
|
97 | signal.addEventListener('abort', abortAndFinalize);
|
98 | }
|
99 |
|
100 | const finalize = () => {
|
101 | request_.abort();
|
102 | if (signal) {
|
103 | signal.removeEventListener('abort', abortAndFinalize);
|
104 | }
|
105 | };
|
106 |
|
107 | request_.on('error', error => {
|
108 | reject(new FetchError(`request to ${request.url} failed, reason: ${error.message}`, 'system', error));
|
109 | finalize();
|
110 | });
|
111 |
|
112 | fixResponseChunkedTransferBadEnding(request_, error => {
|
113 | if (response && response.body) {
|
114 | response.body.destroy(error);
|
115 | }
|
116 | });
|
117 |
|
118 |
|
119 | if (process.version < 'v14') {
|
120 |
|
121 |
|
122 | request_.on('socket', s => {
|
123 | let endedWithEventsCount;
|
124 | s.prependListener('end', () => {
|
125 | endedWithEventsCount = s._eventsCount;
|
126 | });
|
127 | s.prependListener('close', hadError => {
|
128 |
|
129 | if (response && endedWithEventsCount < s._eventsCount && !hadError) {
|
130 | const error = new Error('Premature close');
|
131 | error.code = 'ERR_STREAM_PREMATURE_CLOSE';
|
132 | response.body.emit('error', error);
|
133 | }
|
134 | });
|
135 | });
|
136 | }
|
137 |
|
138 | request_.on('response', response_ => {
|
139 | request_.setTimeout(0);
|
140 | const headers = fromRawHeaders(response_.rawHeaders);
|
141 |
|
142 |
|
143 | if (isRedirect(response_.statusCode)) {
|
144 |
|
145 | const location = headers.get('Location');
|
146 |
|
147 |
|
148 | let locationURL = null;
|
149 | try {
|
150 | locationURL = location === null ? null : new URL(location, request.url);
|
151 | } catch {
|
152 |
|
153 |
|
154 |
|
155 | if (request.redirect !== 'manual') {
|
156 | reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect'));
|
157 | finalize();
|
158 | return;
|
159 | }
|
160 | }
|
161 |
|
162 |
|
163 | switch (request.redirect) {
|
164 | case 'error':
|
165 | reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${request.url}`, 'no-redirect'));
|
166 | finalize();
|
167 | return;
|
168 | case 'manual':
|
169 |
|
170 | break;
|
171 | case 'follow': {
|
172 |
|
173 | if (locationURL === null) {
|
174 | break;
|
175 | }
|
176 |
|
177 |
|
178 | if (request.counter >= request.follow) {
|
179 | reject(new FetchError(`maximum redirect reached at: ${request.url}`, 'max-redirect'));
|
180 | finalize();
|
181 | return;
|
182 | }
|
183 |
|
184 |
|
185 |
|
186 | const requestOptions = {
|
187 | headers: new Headers(request.headers),
|
188 | follow: request.follow,
|
189 | counter: request.counter + 1,
|
190 | agent: request.agent,
|
191 | compress: request.compress,
|
192 | method: request.method,
|
193 | body: clone(request),
|
194 | signal: request.signal,
|
195 | size: request.size,
|
196 | referrer: request.referrer,
|
197 | referrerPolicy: request.referrerPolicy
|
198 | };
|
199 |
|
200 |
|
201 |
|
202 |
|
203 |
|
204 |
|
205 |
|
206 |
|
207 |
|
208 |
|
209 | if (!isDomainOrSubdomain(request.url, locationURL) || !isSameProtocol(request.url, locationURL)) {
|
210 | for (const name of ['authorization', 'www-authenticate', 'cookie', 'cookie2']) {
|
211 | requestOptions.headers.delete(name);
|
212 | }
|
213 | }
|
214 |
|
215 |
|
216 | if (response_.statusCode !== 303 && request.body && options_.body instanceof Stream.Readable) {
|
217 | reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect'));
|
218 | finalize();
|
219 | return;
|
220 | }
|
221 |
|
222 |
|
223 | if (response_.statusCode === 303 || ((response_.statusCode === 301 || response_.statusCode === 302) && request.method === 'POST')) {
|
224 | requestOptions.method = 'GET';
|
225 | requestOptions.body = undefined;
|
226 | requestOptions.headers.delete('content-length');
|
227 | }
|
228 |
|
229 |
|
230 | const responseReferrerPolicy = parseReferrerPolicyFromHeader(headers);
|
231 | if (responseReferrerPolicy) {
|
232 | requestOptions.referrerPolicy = responseReferrerPolicy;
|
233 | }
|
234 |
|
235 |
|
236 | resolve(fetch(new Request(locationURL, requestOptions)));
|
237 | finalize();
|
238 | return;
|
239 | }
|
240 |
|
241 | default:
|
242 | return reject(new TypeError(`Redirect option '${request.redirect}' is not a valid value of RequestRedirect`));
|
243 | }
|
244 | }
|
245 |
|
246 |
|
247 | if (signal) {
|
248 | response_.once('end', () => {
|
249 | signal.removeEventListener('abort', abortAndFinalize);
|
250 | });
|
251 | }
|
252 |
|
253 | let body = pump(response_, new PassThrough(), error => {
|
254 | if (error) {
|
255 | reject(error);
|
256 | }
|
257 | });
|
258 |
|
259 |
|
260 | if (process.version < 'v12.10') {
|
261 | response_.on('aborted', abortAndFinalize);
|
262 | }
|
263 |
|
264 | const responseOptions = {
|
265 | url: request.url,
|
266 | status: response_.statusCode,
|
267 | statusText: response_.statusMessage,
|
268 | headers,
|
269 | size: request.size,
|
270 | counter: request.counter,
|
271 | highWaterMark: request.highWaterMark
|
272 | };
|
273 |
|
274 |
|
275 | const codings = headers.get('Content-Encoding');
|
276 |
|
277 |
|
278 |
|
279 |
|
280 |
|
281 |
|
282 |
|
283 |
|
284 |
|
285 | if (!request.compress || request.method === 'HEAD' || codings === null || response_.statusCode === 204 || response_.statusCode === 304) {
|
286 | response = new Response(body, responseOptions);
|
287 | resolve(response);
|
288 | return;
|
289 | }
|
290 |
|
291 |
|
292 |
|
293 |
|
294 |
|
295 |
|
296 | const zlibOptions = {
|
297 | flush: zlib.Z_SYNC_FLUSH,
|
298 | finishFlush: zlib.Z_SYNC_FLUSH
|
299 | };
|
300 |
|
301 |
|
302 | if (codings === 'gzip' || codings === 'x-gzip') {
|
303 | body = pump(body, zlib.createGunzip(zlibOptions), error => {
|
304 | if (error) {
|
305 | reject(error);
|
306 | }
|
307 | });
|
308 | response = new Response(body, responseOptions);
|
309 | resolve(response);
|
310 | return;
|
311 | }
|
312 |
|
313 |
|
314 | if (codings === 'deflate' || codings === 'x-deflate') {
|
315 |
|
316 |
|
317 | const raw = pump(response_, new PassThrough(), error => {
|
318 | if (error) {
|
319 | reject(error);
|
320 | }
|
321 | });
|
322 | raw.once('data', chunk => {
|
323 |
|
324 | if ((chunk[0] & 0x0F) === 0x08) {
|
325 | body = pump(body, zlib.createInflate(), error => {
|
326 | if (error) {
|
327 | reject(error);
|
328 | }
|
329 | });
|
330 | } else {
|
331 | body = pump(body, zlib.createInflateRaw(), error => {
|
332 | if (error) {
|
333 | reject(error);
|
334 | }
|
335 | });
|
336 | }
|
337 |
|
338 | response = new Response(body, responseOptions);
|
339 | resolve(response);
|
340 | });
|
341 | raw.once('end', () => {
|
342 |
|
343 |
|
344 | if (!response) {
|
345 | response = new Response(body, responseOptions);
|
346 | resolve(response);
|
347 | }
|
348 | });
|
349 | return;
|
350 | }
|
351 |
|
352 |
|
353 | if (codings === 'br') {
|
354 | body = pump(body, zlib.createBrotliDecompress(), error => {
|
355 | if (error) {
|
356 | reject(error);
|
357 | }
|
358 | });
|
359 | response = new Response(body, responseOptions);
|
360 | resolve(response);
|
361 | return;
|
362 | }
|
363 |
|
364 |
|
365 | response = new Response(body, responseOptions);
|
366 | resolve(response);
|
367 | });
|
368 |
|
369 |
|
370 | writeToStream(request_, request).catch(reject);
|
371 | });
|
372 | }
|
373 |
|
374 | function fixResponseChunkedTransferBadEnding(request, errorCallback) {
|
375 | const LAST_CHUNK = Buffer.from('0\r\n\r\n');
|
376 |
|
377 | let isChunkedTransfer = false;
|
378 | let properLastChunkReceived = false;
|
379 | let previousChunk;
|
380 |
|
381 | request.on('response', response => {
|
382 | const {headers} = response;
|
383 | isChunkedTransfer = headers['transfer-encoding'] === 'chunked' && !headers['content-length'];
|
384 | });
|
385 |
|
386 | request.on('socket', socket => {
|
387 | const onSocketClose = () => {
|
388 | if (isChunkedTransfer && !properLastChunkReceived) {
|
389 | const error = new Error('Premature close');
|
390 | error.code = 'ERR_STREAM_PREMATURE_CLOSE';
|
391 | errorCallback(error);
|
392 | }
|
393 | };
|
394 |
|
395 | const onData = buf => {
|
396 | properLastChunkReceived = Buffer.compare(buf.slice(-5), LAST_CHUNK) === 0;
|
397 |
|
398 |
|
399 | if (!properLastChunkReceived && previousChunk) {
|
400 | properLastChunkReceived = (
|
401 | Buffer.compare(previousChunk.slice(-3), LAST_CHUNK.slice(0, 3)) === 0 &&
|
402 | Buffer.compare(buf.slice(-2), LAST_CHUNK.slice(3)) === 0
|
403 | );
|
404 | }
|
405 |
|
406 | previousChunk = buf;
|
407 | };
|
408 |
|
409 | socket.prependListener('close', onSocketClose);
|
410 | socket.on('data', onData);
|
411 |
|
412 | request.on('close', () => {
|
413 | socket.removeListener('close', onSocketClose);
|
414 | socket.removeListener('data', onData);
|
415 | });
|
416 | });
|
417 | }
|