1 | 'use strict';
|
2 | const EventEmitter = require('events');
|
3 | const http = require('http');
|
4 | const https = require('https');
|
5 | const PassThrough = require('stream').PassThrough;
|
6 | const Transform = require('stream').Transform;
|
7 | const urlLib = require('url');
|
8 | const fs = require('fs');
|
9 | const querystring = require('querystring');
|
10 | const CacheableRequest = require('cacheable-request');
|
11 | const duplexer3 = require('duplexer3');
|
12 | const intoStream = require('into-stream');
|
13 | const is = require('@sindresorhus/is');
|
14 | const getStream = require('get-stream');
|
15 | const timedOut = require('timed-out');
|
16 | const urlParseLax = require('url-parse-lax');
|
17 | const urlToOptions = require('url-to-options');
|
18 | const lowercaseKeys = require('lowercase-keys');
|
19 | const decompressResponse = require('decompress-response');
|
20 | const mimicResponse = require('mimic-response');
|
21 | const isRetryAllowed = require('is-retry-allowed');
|
22 | const isURL = require('isurl');
|
23 | const PCancelable = require('p-cancelable');
|
24 | const pTimeout = require('p-timeout');
|
25 | const pify = require('pify');
|
26 | const Buffer = require('safe-buffer').Buffer;
|
27 | const pkg = require('./package.json');
|
28 | const errors = require('./errors');
|
29 |
|
30 | const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);
|
31 | const allMethodRedirectCodes = new Set([300, 303, 307, 308]);
|
32 |
|
33 | const isFormData = body => is.nodeStream(body) && is.function(body.getBoundary);
|
34 |
|
35 | const getBodySize = opts => {
|
36 | const body = opts.body;
|
37 |
|
38 | if (opts.headers['content-length']) {
|
39 | return Number(opts.headers['content-length']);
|
40 | }
|
41 |
|
42 | if (!body && !opts.stream) {
|
43 | return 0;
|
44 | }
|
45 |
|
46 | if (is.string(body)) {
|
47 | return Buffer.byteLength(body);
|
48 | }
|
49 |
|
50 | if (isFormData(body)) {
|
51 | return pify(body.getLength.bind(body))();
|
52 | }
|
53 |
|
54 | if (body instanceof fs.ReadStream) {
|
55 | return pify(fs.stat)(body.path).then(stat => stat.size);
|
56 | }
|
57 |
|
58 | if (is.nodeStream(body) && is.buffer(body._buffer)) {
|
59 | return body._buffer.length;
|
60 | }
|
61 |
|
62 | return null;
|
63 | };
|
64 |
|
65 | function requestAsEventEmitter(opts) {
|
66 | opts = opts || {};
|
67 |
|
68 | const ee = new EventEmitter();
|
69 | const requestUrl = opts.href || urlLib.resolve(urlLib.format(opts), opts.path);
|
70 | const redirects = [];
|
71 | const agents = is.object(opts.agent) ? opts.agent : null;
|
72 | let retryCount = 0;
|
73 | let redirectUrl;
|
74 | let uploadBodySize;
|
75 | let uploaded = 0;
|
76 |
|
77 | const get = opts => {
|
78 | if (opts.protocol !== 'http:' && opts.protocol !== 'https:') {
|
79 | ee.emit('error', new got.UnsupportedProtocolError(opts));
|
80 | return;
|
81 | }
|
82 |
|
83 | let fn = opts.protocol === 'https:' ? https : http;
|
84 |
|
85 | if (agents) {
|
86 | const protocolName = opts.protocol === 'https:' ? 'https' : 'http';
|
87 | opts.agent = agents[protocolName] || opts.agent;
|
88 | }
|
89 |
|
90 | if (opts.useElectronNet && process.versions.electron) {
|
91 | const electron = require('electron');
|
92 | fn = electron.net || electron.remote.net;
|
93 | }
|
94 |
|
95 | let progressInterval;
|
96 |
|
97 | const cacheableRequest = new CacheableRequest(fn.request, opts.cache);
|
98 | const cacheReq = cacheableRequest(opts, res => {
|
99 | clearInterval(progressInterval);
|
100 |
|
101 | ee.emit('uploadProgress', {
|
102 | percent: 1,
|
103 | transferred: uploaded,
|
104 | total: uploadBodySize
|
105 | });
|
106 |
|
107 | const statusCode = res.statusCode;
|
108 |
|
109 | res.url = redirectUrl || requestUrl;
|
110 | res.requestUrl = requestUrl;
|
111 |
|
112 | const followRedirect = opts.followRedirect && 'location' in res.headers;
|
113 | const redirectGet = followRedirect && getMethodRedirectCodes.has(statusCode);
|
114 | const redirectAll = followRedirect && allMethodRedirectCodes.has(statusCode);
|
115 |
|
116 | if (redirectAll || (redirectGet && (opts.method === 'GET' || opts.method === 'HEAD'))) {
|
117 | res.resume();
|
118 |
|
119 | if (statusCode === 303) {
|
120 |
|
121 |
|
122 | opts.method = 'GET';
|
123 | }
|
124 |
|
125 | if (redirects.length >= 10) {
|
126 | ee.emit('error', new got.MaxRedirectsError(statusCode, redirects, opts), null, res);
|
127 | return;
|
128 | }
|
129 |
|
130 | const bufferString = Buffer.from(res.headers.location, 'binary').toString();
|
131 |
|
132 | redirectUrl = urlLib.resolve(urlLib.format(opts), bufferString);
|
133 |
|
134 | redirects.push(redirectUrl);
|
135 |
|
136 | const redirectOpts = Object.assign({}, opts, urlLib.parse(redirectUrl));
|
137 |
|
138 | ee.emit('redirect', res, redirectOpts);
|
139 |
|
140 | get(redirectOpts);
|
141 |
|
142 | return;
|
143 | }
|
144 |
|
145 | setImmediate(() => {
|
146 | try {
|
147 | getResponse(res, opts, ee, redirects);
|
148 | } catch (e) {
|
149 | ee.emit('error', e);
|
150 | }
|
151 | });
|
152 | });
|
153 |
|
154 | cacheReq.on('error', err => {
|
155 | if (err instanceof CacheableRequest.RequestError) {
|
156 | ee.emit('error', new got.RequestError(err, opts));
|
157 | } else {
|
158 | ee.emit('error', new got.CacheError(err, opts));
|
159 | }
|
160 | });
|
161 |
|
162 | cacheReq.once('request', req => {
|
163 | let aborted = false;
|
164 | req.once('abort', _ => {
|
165 | aborted = true;
|
166 | });
|
167 |
|
168 | req.once('error', err => {
|
169 | clearInterval(progressInterval);
|
170 |
|
171 | if (aborted) {
|
172 | return;
|
173 | }
|
174 |
|
175 | const backoff = opts.retries(++retryCount, err);
|
176 |
|
177 | if (backoff) {
|
178 | setTimeout(get, backoff, opts);
|
179 | return;
|
180 | }
|
181 |
|
182 | ee.emit('error', new got.RequestError(err, opts));
|
183 | });
|
184 |
|
185 | ee.once('request', req => {
|
186 | ee.emit('uploadProgress', {
|
187 | percent: 0,
|
188 | transferred: 0,
|
189 | total: uploadBodySize
|
190 | });
|
191 |
|
192 | const socket = req.connection;
|
193 | if (socket) {
|
194 |
|
195 | const isConnecting = socket.connecting === undefined ? socket._connecting : socket.connecting;
|
196 |
|
197 | const onSocketConnect = () => {
|
198 | const uploadEventFrequency = 150;
|
199 |
|
200 | progressInterval = setInterval(() => {
|
201 | const lastUploaded = uploaded;
|
202 | const headersSize = Buffer.byteLength(req._header);
|
203 | uploaded = socket.bytesWritten - headersSize;
|
204 |
|
205 |
|
206 | if (uploadBodySize && uploaded > uploadBodySize) {
|
207 | uploaded = uploadBodySize;
|
208 | }
|
209 |
|
210 |
|
211 |
|
212 |
|
213 | if (uploaded === lastUploaded || uploaded === uploadBodySize) {
|
214 | return;
|
215 | }
|
216 |
|
217 | ee.emit('uploadProgress', {
|
218 | percent: uploadBodySize ? uploaded / uploadBodySize : 0,
|
219 | transferred: uploaded,
|
220 | total: uploadBodySize
|
221 | });
|
222 | }, uploadEventFrequency);
|
223 | };
|
224 |
|
225 |
|
226 |
|
227 |
|
228 |
|
229 | if (isConnecting) {
|
230 | socket.once('connect', onSocketConnect);
|
231 | } else {
|
232 | onSocketConnect();
|
233 | }
|
234 | }
|
235 | });
|
236 |
|
237 | if (opts.gotTimeout) {
|
238 | clearInterval(progressInterval);
|
239 | timedOut(req, opts.gotTimeout);
|
240 | }
|
241 |
|
242 | setImmediate(() => {
|
243 | ee.emit('request', req);
|
244 | });
|
245 | });
|
246 | };
|
247 |
|
248 | setImmediate(() => {
|
249 | Promise.resolve(getBodySize(opts))
|
250 | .then(size => {
|
251 | uploadBodySize = size;
|
252 | get(opts);
|
253 | })
|
254 | .catch(err => {
|
255 | ee.emit('error', err);
|
256 | });
|
257 | });
|
258 |
|
259 | return ee;
|
260 | }
|
261 |
|
262 | function getResponse(res, opts, ee, redirects) {
|
263 | const downloadBodySize = Number(res.headers['content-length']) || null;
|
264 | let downloaded = 0;
|
265 |
|
266 | const progressStream = new Transform({
|
267 | transform(chunk, encoding, callback) {
|
268 | downloaded += chunk.length;
|
269 |
|
270 | const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
|
271 |
|
272 |
|
273 | if (percent < 1) {
|
274 | ee.emit('downloadProgress', {
|
275 | percent,
|
276 | transferred: downloaded,
|
277 | total: downloadBodySize
|
278 | });
|
279 | }
|
280 |
|
281 | callback(null, chunk);
|
282 | },
|
283 |
|
284 | flush(callback) {
|
285 | ee.emit('downloadProgress', {
|
286 | percent: 1,
|
287 | transferred: downloaded,
|
288 | total: downloadBodySize
|
289 | });
|
290 |
|
291 | callback();
|
292 | }
|
293 | });
|
294 |
|
295 | mimicResponse(res, progressStream);
|
296 | progressStream.redirectUrls = redirects;
|
297 |
|
298 | const response = opts.decompress === true &&
|
299 | is.function(decompressResponse) &&
|
300 | opts.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream;
|
301 |
|
302 | if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) {
|
303 | opts.encoding = null;
|
304 | }
|
305 |
|
306 | ee.emit('response', response);
|
307 |
|
308 | ee.emit('downloadProgress', {
|
309 | percent: 0,
|
310 | transferred: 0,
|
311 | total: downloadBodySize
|
312 | });
|
313 |
|
314 | res.pipe(progressStream);
|
315 | }
|
316 |
|
317 | function asPromise(opts) {
|
318 | const timeoutFn = requestPromise => opts.gotTimeout && opts.gotTimeout.request ?
|
319 | pTimeout(requestPromise, opts.gotTimeout.request, new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts)) :
|
320 | requestPromise;
|
321 |
|
322 | const proxy = new EventEmitter();
|
323 |
|
324 | const cancelable = new PCancelable((resolve, reject, onCancel) => {
|
325 | const ee = requestAsEventEmitter(opts);
|
326 | let cancelOnRequest = false;
|
327 |
|
328 | onCancel(() => {
|
329 | cancelOnRequest = true;
|
330 | });
|
331 |
|
332 | ee.on('request', req => {
|
333 | if (cancelOnRequest) {
|
334 | req.abort();
|
335 | }
|
336 |
|
337 | onCancel(() => {
|
338 | req.abort();
|
339 | });
|
340 |
|
341 | if (is.nodeStream(opts.body)) {
|
342 | opts.body.pipe(req);
|
343 | opts.body = undefined;
|
344 | return;
|
345 | }
|
346 |
|
347 | req.end(opts.body);
|
348 | });
|
349 |
|
350 | ee.on('response', res => {
|
351 | const stream = is.null(opts.encoding) ? getStream.buffer(res) : getStream(res, opts);
|
352 |
|
353 | stream
|
354 | .catch(err => reject(new got.ReadError(err, opts)))
|
355 | .then(data => {
|
356 | const statusCode = res.statusCode;
|
357 | const limitStatusCode = opts.followRedirect ? 299 : 399;
|
358 |
|
359 | res.body = data;
|
360 |
|
361 | if (opts.json && res.body) {
|
362 | try {
|
363 | res.body = JSON.parse(res.body);
|
364 | } catch (err) {
|
365 | if (statusCode >= 200 && statusCode < 300) {
|
366 | throw new got.ParseError(err, statusCode, opts, data);
|
367 | }
|
368 | }
|
369 | }
|
370 |
|
371 | if (opts.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) {
|
372 | throw new got.HTTPError(statusCode, res.statusMessage, res.headers, opts);
|
373 | }
|
374 |
|
375 | resolve(res);
|
376 | })
|
377 | .catch(err => {
|
378 | Object.defineProperty(err, 'response', {value: res});
|
379 | reject(err);
|
380 | });
|
381 | });
|
382 |
|
383 | ee.once('error', reject);
|
384 | ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
|
385 | ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
|
386 | ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
|
387 | });
|
388 |
|
389 |
|
390 |
|
391 | Object.defineProperty(cancelable, 'canceled', {
|
392 | get() {
|
393 | return cancelable.isCanceled;
|
394 | }
|
395 | });
|
396 |
|
397 | const promise = timeoutFn(cancelable);
|
398 |
|
399 | promise.cancel = cancelable.cancel.bind(cancelable);
|
400 |
|
401 | promise.on = (name, fn) => {
|
402 | proxy.on(name, fn);
|
403 | return promise;
|
404 | };
|
405 |
|
406 | return promise;
|
407 | }
|
408 |
|
409 | function asStream(opts) {
|
410 | opts.stream = true;
|
411 |
|
412 | const input = new PassThrough();
|
413 | const output = new PassThrough();
|
414 | const proxy = duplexer3(input, output);
|
415 | let timeout;
|
416 |
|
417 | if (opts.gotTimeout && opts.gotTimeout.request) {
|
418 | timeout = setTimeout(() => {
|
419 | proxy.emit('error', new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts));
|
420 | }, opts.gotTimeout.request);
|
421 | }
|
422 |
|
423 | if (opts.json) {
|
424 | throw new Error('Got can not be used as a stream when the `json` option is used');
|
425 | }
|
426 |
|
427 | if (opts.body) {
|
428 | proxy.write = () => {
|
429 | throw new Error('Got\'s stream is not writable when the `body` option is used');
|
430 | };
|
431 | }
|
432 |
|
433 | const ee = requestAsEventEmitter(opts);
|
434 |
|
435 | ee.on('request', req => {
|
436 | proxy.emit('request', req);
|
437 |
|
438 | if (is.nodeStream(opts.body)) {
|
439 | opts.body.pipe(req);
|
440 | return;
|
441 | }
|
442 |
|
443 | if (opts.body) {
|
444 | req.end(opts.body);
|
445 | return;
|
446 | }
|
447 |
|
448 | if (opts.method === 'POST' || opts.method === 'PUT' || opts.method === 'PATCH') {
|
449 | input.pipe(req);
|
450 | return;
|
451 | }
|
452 |
|
453 | req.end();
|
454 | });
|
455 |
|
456 | ee.on('response', res => {
|
457 | clearTimeout(timeout);
|
458 |
|
459 | const statusCode = res.statusCode;
|
460 |
|
461 | res.on('error', err => {
|
462 | proxy.emit('error', new got.ReadError(err, opts));
|
463 | });
|
464 |
|
465 | res.pipe(output);
|
466 |
|
467 | if (opts.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
|
468 | proxy.emit('error', new got.HTTPError(statusCode, res.statusMessage, res.headers, opts), null, res);
|
469 | return;
|
470 | }
|
471 |
|
472 | proxy.emit('response', res);
|
473 | });
|
474 |
|
475 | ee.on('error', proxy.emit.bind(proxy, 'error'));
|
476 | ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
|
477 | ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
|
478 | ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
|
479 |
|
480 | return proxy;
|
481 | }
|
482 |
|
483 | function normalizeArguments(url, opts) {
|
484 | if (!is.string(url) && !is.object(url)) {
|
485 | throw new TypeError(`Parameter \`url\` must be a string or object, not ${is(url)}`);
|
486 | } else if (is.string(url)) {
|
487 | url = url.replace(/^unix:/, 'http://$&');
|
488 |
|
489 | try {
|
490 | decodeURI(url);
|
491 | } catch (err) {
|
492 | throw new Error('Parameter `url` must contain valid UTF-8 character sequences');
|
493 | }
|
494 |
|
495 | url = urlParseLax(url);
|
496 | if (url.auth) {
|
497 | throw new Error('Basic authentication must be done with the `auth` option');
|
498 | }
|
499 | } else if (isURL.lenient(url)) {
|
500 | url = urlToOptions(url);
|
501 | }
|
502 |
|
503 | opts = Object.assign(
|
504 | {
|
505 | path: '',
|
506 | retries: 2,
|
507 | cache: false,
|
508 | decompress: true,
|
509 | useElectronNet: false,
|
510 | throwHttpErrors: true
|
511 | },
|
512 | url,
|
513 | {
|
514 | protocol: url.protocol || 'http:'
|
515 | },
|
516 | opts
|
517 | );
|
518 |
|
519 | const headers = lowercaseKeys(opts.headers);
|
520 | for (const key of Object.keys(headers)) {
|
521 | if (is.nullOrUndefined(headers[key])) {
|
522 | delete headers[key];
|
523 | }
|
524 | }
|
525 |
|
526 | opts.headers = Object.assign({
|
527 | 'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`
|
528 | }, headers);
|
529 |
|
530 | if (opts.decompress) {
|
531 | opts.headers['accept-encoding'] = 'gzip,deflate';
|
532 | }
|
533 |
|
534 | const query = opts.query;
|
535 |
|
536 | if (query) {
|
537 | if (!is.string(query)) {
|
538 | opts.query = querystring.stringify(query);
|
539 | }
|
540 |
|
541 | opts.path = `${opts.path.split('?')[0]}?${opts.query}`;
|
542 | delete opts.query;
|
543 | }
|
544 |
|
545 | if (opts.json && is.undefined(opts.headers.accept)) {
|
546 | opts.headers.accept = 'application/json';
|
547 | }
|
548 |
|
549 | const body = opts.body;
|
550 | if (is.nullOrUndefined(body)) {
|
551 | opts.method = (opts.method || 'GET').toUpperCase();
|
552 | } else {
|
553 | const headers = opts.headers;
|
554 | if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body) && !(opts.form || opts.json)) {
|
555 | throw new TypeError('The `body` option must be a stream.Readable, string, Buffer or plain Object');
|
556 | }
|
557 |
|
558 | const canBodyBeStringified = is.plainObject(body) || is.array(body);
|
559 | if ((opts.form || opts.json) && !canBodyBeStringified) {
|
560 | throw new TypeError('The `body` option must be a plain Object or Array when the `form` or `json` option is used');
|
561 | }
|
562 |
|
563 | if (isFormData(body)) {
|
564 |
|
565 | headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
|
566 | } else if (opts.form && canBodyBeStringified) {
|
567 | headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
|
568 | opts.body = querystring.stringify(body);
|
569 | } else if (opts.json && canBodyBeStringified) {
|
570 | headers['content-type'] = headers['content-type'] || 'application/json';
|
571 | opts.body = JSON.stringify(body);
|
572 | }
|
573 |
|
574 | if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding']) && !is.nodeStream(body)) {
|
575 | const length = is.string(opts.body) ? Buffer.byteLength(opts.body) : opts.body.length;
|
576 | headers['content-length'] = length;
|
577 | }
|
578 |
|
579 |
|
580 |
|
581 | if (is.buffer(body)) {
|
582 | opts.body = intoStream(body);
|
583 | opts.body._buffer = body;
|
584 | }
|
585 |
|
586 | opts.method = (opts.method || 'POST').toUpperCase();
|
587 | }
|
588 |
|
589 | if (opts.hostname === 'unix') {
|
590 | const matches = /(.+?):(.+)/.exec(opts.path);
|
591 |
|
592 | if (matches) {
|
593 | opts.socketPath = matches[1];
|
594 | opts.path = matches[2];
|
595 | opts.host = null;
|
596 | }
|
597 | }
|
598 |
|
599 | if (!is.function(opts.retries)) {
|
600 | const retries = opts.retries;
|
601 |
|
602 | opts.retries = (iter, err) => {
|
603 | if (iter > retries || !isRetryAllowed(err)) {
|
604 | return 0;
|
605 | }
|
606 |
|
607 | const noise = Math.random() * 100;
|
608 |
|
609 | return ((1 << iter) * 1000) + noise;
|
610 | };
|
611 | }
|
612 |
|
613 | if (is.undefined(opts.followRedirect)) {
|
614 | opts.followRedirect = true;
|
615 | }
|
616 |
|
617 | if (opts.timeout) {
|
618 | if (is.number(opts.timeout)) {
|
619 | opts.gotTimeout = {request: opts.timeout};
|
620 | } else {
|
621 | opts.gotTimeout = opts.timeout;
|
622 | }
|
623 | delete opts.timeout;
|
624 | }
|
625 |
|
626 | return opts;
|
627 | }
|
628 |
|
629 | function got(url, opts) {
|
630 | try {
|
631 | const normalizedArgs = normalizeArguments(url, opts);
|
632 |
|
633 | if (normalizedArgs.stream) {
|
634 | return asStream(normalizedArgs);
|
635 | }
|
636 |
|
637 | return asPromise(normalizedArgs);
|
638 | } catch (err) {
|
639 | return Promise.reject(err);
|
640 | }
|
641 | }
|
642 |
|
643 | got.stream = (url, opts) => asStream(normalizeArguments(url, opts));
|
644 |
|
645 | const methods = [
|
646 | 'get',
|
647 | 'post',
|
648 | 'put',
|
649 | 'patch',
|
650 | 'head',
|
651 | 'delete'
|
652 | ];
|
653 |
|
654 | for (const method of methods) {
|
655 | got[method] = (url, opts) => got(url, Object.assign({}, opts, {method}));
|
656 | got.stream[method] = (url, opts) => got.stream(url, Object.assign({}, opts, {method}));
|
657 | }
|
658 |
|
659 | Object.assign(got, errors);
|
660 |
|
661 | module.exports = got;
|