1 | 'use strict';
|
2 | const EventEmitter = require('events');
|
3 | const http = require('http');
|
4 | const https = require('https');
|
5 | const PassThrough = require('stream').PassThrough;
|
6 | const Transform = require('stream').Transform;
|
7 | const urlLib = require('url');
|
8 | const fs = require('fs');
|
9 | const querystring = require('querystring');
|
10 | const CacheableRequest = require('cacheable-request');
|
11 | const duplexer3 = require('duplexer3');
|
12 | const intoStream = require('into-stream');
|
13 | const is = require('@sindresorhus/is');
|
14 | const getStream = require('get-stream');
|
15 | const timedOut = require('timed-out');
|
16 | const urlParseLax = require('url-parse-lax');
|
17 | const urlToOptions = require('url-to-options');
|
18 | const lowercaseKeys = require('lowercase-keys');
|
19 | const decompressResponse = require('decompress-response');
|
20 | const mimicResponse = require('mimic-response');
|
21 | const isRetryAllowed = require('is-retry-allowed');
|
22 | const isURL = require('isurl');
|
23 | const PCancelable = require('p-cancelable');
|
24 | const pTimeout = require('p-timeout');
|
25 | const pify = require('pify');
|
26 | const Buffer = require('safe-buffer').Buffer;
|
27 | const pkg = require('./package.json');
|
28 | const errors = require('./errors');
|
29 |
|
30 | const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);
|
31 | const allMethodRedirectCodes = new Set([300, 303, 307, 308]);
|
32 |
|
33 | const isFormData = body => is.nodeStream(body) && is.function(body.getBoundary);
|
34 |
|
35 | const getBodySize = opts => {
|
36 | const body = opts.body;
|
37 |
|
38 | if (opts.headers['content-length']) {
|
39 | return Number(opts.headers['content-length']);
|
40 | }
|
41 |
|
42 | if (!body && !opts.stream) {
|
43 | return 0;
|
44 | }
|
45 |
|
46 | if (is.string(body)) {
|
47 | return Buffer.byteLength(body);
|
48 | }
|
49 |
|
50 | if (isFormData(body)) {
|
51 | return pify(body.getLength.bind(body))();
|
52 | }
|
53 |
|
54 | if (body instanceof fs.ReadStream) {
|
55 | return pify(fs.stat)(body.path).then(stat => stat.size);
|
56 | }
|
57 |
|
58 | if (is.nodeStream(body) && is.buffer(body._buffer)) {
|
59 | return body._buffer.length;
|
60 | }
|
61 |
|
62 | return null;
|
63 | };
|
64 |
|
65 | function requestAsEventEmitter(opts) {
|
66 | opts = opts || {};
|
67 |
|
68 | const ee = new EventEmitter();
|
69 | const requestUrl = opts.href || urlLib.resolve(urlLib.format(opts), opts.path);
|
70 | const redirects = [];
|
71 | const agents = is.object(opts.agent) ? opts.agent : null;
|
72 | let retryCount = 0;
|
73 | let redirectUrl;
|
74 | let uploadBodySize;
|
75 | let uploaded = 0;
|
76 |
|
77 | const get = opts => {
|
78 | if (opts.protocol !== 'http:' && opts.protocol !== 'https:') {
|
79 | ee.emit('error', new got.UnsupportedProtocolError(opts));
|
80 | return;
|
81 | }
|
82 |
|
83 | let fn = opts.protocol === 'https:' ? https : http;
|
84 |
|
85 | if (agents) {
|
86 | const protocolName = opts.protocol === 'https:' ? 'https' : 'http';
|
87 | opts.agent = agents[protocolName] || opts.agent;
|
88 | }
|
89 |
|
90 | if (opts.useElectronNet && process.versions.electron) {
|
91 | const electron = require('electron');
|
92 | fn = electron.net || electron.remote.net;
|
93 | }
|
94 |
|
95 | let progressInterval;
|
96 |
|
97 | const cacheableRequest = new CacheableRequest(fn.request, opts.cache);
|
98 | const cacheReq = cacheableRequest(opts, res => {
|
99 | clearInterval(progressInterval);
|
100 |
|
101 | ee.emit('uploadProgress', {
|
102 | percent: 1,
|
103 | transferred: uploaded,
|
104 | total: uploadBodySize
|
105 | });
|
106 |
|
107 | const statusCode = res.statusCode;
|
108 |
|
109 | res.url = redirectUrl || requestUrl;
|
110 | res.requestUrl = requestUrl;
|
111 |
|
112 | const followRedirect = opts.followRedirect && 'location' in res.headers;
|
113 | const redirectGet = followRedirect && getMethodRedirectCodes.has(statusCode);
|
114 | const redirectAll = followRedirect && allMethodRedirectCodes.has(statusCode);
|
115 |
|
116 | if (redirectAll || (redirectGet && (opts.method === 'GET' || opts.method === 'HEAD'))) {
|
117 | res.resume();
|
118 |
|
119 | if (statusCode === 303) {
|
120 |
|
121 |
|
122 | opts.method = 'GET';
|
123 | }
|
124 |
|
125 | if (redirects.length >= 10) {
|
126 | ee.emit('error', new got.MaxRedirectsError(statusCode, redirects, opts), null, res);
|
127 | return;
|
128 | }
|
129 |
|
130 | const bufferString = Buffer.from(res.headers.location, 'binary').toString();
|
131 |
|
132 | redirectUrl = urlLib.resolve(urlLib.format(opts), bufferString);
|
133 |
|
134 | redirects.push(redirectUrl);
|
135 |
|
136 | const redirectOpts = Object.assign({}, opts, urlLib.parse(redirectUrl));
|
137 |
|
138 | ee.emit('redirect', res, redirectOpts);
|
139 |
|
140 | get(redirectOpts);
|
141 |
|
142 | return;
|
143 | }
|
144 |
|
145 | const downloadBodySize = Number(res.headers['content-length']) || null;
|
146 | let downloaded = 0;
|
147 |
|
148 | setImmediate(() => {
|
149 | const progressStream = new Transform({
|
150 | transform(chunk, encoding, callback) {
|
151 | downloaded += chunk.length;
|
152 |
|
153 | const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
|
154 |
|
155 |
|
156 | if (percent < 1) {
|
157 | ee.emit('downloadProgress', {
|
158 | percent,
|
159 | transferred: downloaded,
|
160 | total: downloadBodySize
|
161 | });
|
162 | }
|
163 |
|
164 | callback(null, chunk);
|
165 | },
|
166 |
|
167 | flush(callback) {
|
168 | ee.emit('downloadProgress', {
|
169 | percent: 1,
|
170 | transferred: downloaded,
|
171 | total: downloadBodySize
|
172 | });
|
173 |
|
174 | callback();
|
175 | }
|
176 | });
|
177 |
|
178 | mimicResponse(res, progressStream);
|
179 | progressStream.redirectUrls = redirects;
|
180 |
|
181 | const response = opts.decompress === true &&
|
182 | is.function(decompressResponse) &&
|
183 | opts.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream;
|
184 |
|
185 | if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) {
|
186 | opts.encoding = null;
|
187 | }
|
188 |
|
189 | ee.emit('response', response);
|
190 |
|
191 | ee.emit('downloadProgress', {
|
192 | percent: 0,
|
193 | transferred: 0,
|
194 | total: downloadBodySize
|
195 | });
|
196 |
|
197 | res.pipe(progressStream);
|
198 | });
|
199 | });
|
200 |
|
201 | cacheReq.on('error', err => {
|
202 | if (err instanceof CacheableRequest.RequestError) {
|
203 | ee.emit('error', new got.RequestError(err, opts));
|
204 | } else {
|
205 | ee.emit('error', new got.CacheError(err, opts));
|
206 | }
|
207 | });
|
208 |
|
209 | cacheReq.once('request', req => {
|
210 | req.once('error', err => {
|
211 | clearInterval(progressInterval);
|
212 |
|
213 | const backoff = opts.retries(++retryCount, err);
|
214 |
|
215 | if (backoff) {
|
216 | setTimeout(get, backoff, opts);
|
217 | return;
|
218 | }
|
219 |
|
220 | ee.emit('error', new got.RequestError(err, opts));
|
221 | });
|
222 |
|
223 | ee.once('request', req => {
|
224 | ee.emit('uploadProgress', {
|
225 | percent: 0,
|
226 | transferred: 0,
|
227 | total: uploadBodySize
|
228 | });
|
229 |
|
230 | if (req.connection) {
|
231 | req.connection.once('connect', () => {
|
232 | const uploadEventFrequency = 150;
|
233 |
|
234 | progressInterval = setInterval(() => {
|
235 | const lastUploaded = uploaded;
|
236 | const headersSize = Buffer.byteLength(req._header);
|
237 | uploaded = req.connection.bytesWritten - headersSize;
|
238 |
|
239 |
|
240 | if (uploadBodySize && uploaded > uploadBodySize) {
|
241 | uploaded = uploadBodySize;
|
242 | }
|
243 |
|
244 |
|
245 |
|
246 |
|
247 | if (uploaded === lastUploaded || uploaded === uploadBodySize) {
|
248 | return;
|
249 | }
|
250 |
|
251 | ee.emit('uploadProgress', {
|
252 | percent: uploadBodySize ? uploaded / uploadBodySize : 0,
|
253 | transferred: uploaded,
|
254 | total: uploadBodySize
|
255 | });
|
256 | }, uploadEventFrequency);
|
257 | });
|
258 | }
|
259 | });
|
260 |
|
261 | if (opts.gotTimeout) {
|
262 | clearInterval(progressInterval);
|
263 | timedOut(req, opts.gotTimeout);
|
264 | }
|
265 |
|
266 | setImmediate(() => {
|
267 | ee.emit('request', req);
|
268 | });
|
269 | });
|
270 | };
|
271 |
|
272 | setImmediate(() => {
|
273 | Promise.resolve(getBodySize(opts))
|
274 | .then(size => {
|
275 | uploadBodySize = size;
|
276 | get(opts);
|
277 | })
|
278 | .catch(err => {
|
279 | ee.emit('error', err);
|
280 | });
|
281 | });
|
282 |
|
283 | return ee;
|
284 | }
|
285 |
|
286 | function asPromise(opts) {
|
287 | const timeoutFn = requestPromise => opts.gotTimeout && opts.gotTimeout.request ?
|
288 | pTimeout(requestPromise, opts.gotTimeout.request, new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts)) :
|
289 | requestPromise;
|
290 |
|
291 | const proxy = new EventEmitter();
|
292 |
|
293 | const cancelable = new PCancelable((onCancel, resolve, reject) => {
|
294 | const ee = requestAsEventEmitter(opts);
|
295 | let cancelOnRequest = false;
|
296 |
|
297 | onCancel(() => {
|
298 | cancelOnRequest = true;
|
299 | });
|
300 |
|
301 | ee.on('request', req => {
|
302 | if (cancelOnRequest) {
|
303 | req.abort();
|
304 | }
|
305 |
|
306 | onCancel(() => {
|
307 | req.abort();
|
308 | });
|
309 |
|
310 | if (is.nodeStream(opts.body)) {
|
311 | opts.body.pipe(req);
|
312 | opts.body = undefined;
|
313 | return;
|
314 | }
|
315 |
|
316 | req.end(opts.body);
|
317 | });
|
318 |
|
319 | ee.on('response', res => {
|
320 | const stream = is.null(opts.encoding) ? getStream.buffer(res) : getStream(res, opts);
|
321 |
|
322 | stream
|
323 | .catch(err => reject(new got.ReadError(err, opts)))
|
324 | .then(data => {
|
325 | const statusCode = res.statusCode;
|
326 | const limitStatusCode = opts.followRedirect ? 299 : 399;
|
327 |
|
328 | res.body = data;
|
329 |
|
330 | if (opts.json && res.body) {
|
331 | try {
|
332 | res.body = JSON.parse(res.body);
|
333 | } catch (err) {
|
334 | if (statusCode >= 200 && statusCode < 300) {
|
335 | throw new got.ParseError(err, statusCode, opts, data);
|
336 | }
|
337 | }
|
338 | }
|
339 |
|
340 | if (opts.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) {
|
341 | throw new got.HTTPError(statusCode, res.statusMessage, res.headers, opts);
|
342 | }
|
343 |
|
344 | resolve(res);
|
345 | })
|
346 | .catch(err => {
|
347 | Object.defineProperty(err, 'response', {value: res});
|
348 | reject(err);
|
349 | });
|
350 | });
|
351 |
|
352 | ee.once('error', reject);
|
353 | ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
|
354 | ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
|
355 | ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
|
356 | });
|
357 |
|
358 | const promise = timeoutFn(cancelable);
|
359 |
|
360 | promise.cancel = cancelable.cancel.bind(cancelable);
|
361 |
|
362 | promise.on = (name, fn) => {
|
363 | proxy.on(name, fn);
|
364 | return promise;
|
365 | };
|
366 |
|
367 | return promise;
|
368 | }
|
369 |
|
370 | function asStream(opts) {
|
371 | opts.stream = true;
|
372 |
|
373 | const input = new PassThrough();
|
374 | const output = new PassThrough();
|
375 | const proxy = duplexer3(input, output);
|
376 | let timeout;
|
377 |
|
378 | if (opts.gotTimeout && opts.gotTimeout.request) {
|
379 | timeout = setTimeout(() => {
|
380 | proxy.emit('error', new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts));
|
381 | }, opts.gotTimeout.request);
|
382 | }
|
383 |
|
384 | if (opts.json) {
|
385 | throw new Error('Got can not be used as a stream when the `json` option is used');
|
386 | }
|
387 |
|
388 | if (opts.body) {
|
389 | proxy.write = () => {
|
390 | throw new Error('Got\'s stream is not writable when the `body` option is used');
|
391 | };
|
392 | }
|
393 |
|
394 | const ee = requestAsEventEmitter(opts);
|
395 |
|
396 | ee.on('request', req => {
|
397 | proxy.emit('request', req);
|
398 |
|
399 | if (is.nodeStream(opts.body)) {
|
400 | opts.body.pipe(req);
|
401 | return;
|
402 | }
|
403 |
|
404 | if (opts.body) {
|
405 | req.end(opts.body);
|
406 | return;
|
407 | }
|
408 |
|
409 | if (opts.method === 'POST' || opts.method === 'PUT' || opts.method === 'PATCH') {
|
410 | input.pipe(req);
|
411 | return;
|
412 | }
|
413 |
|
414 | req.end();
|
415 | });
|
416 |
|
417 | ee.on('response', res => {
|
418 | clearTimeout(timeout);
|
419 |
|
420 | const statusCode = res.statusCode;
|
421 |
|
422 | res.on('error', err => {
|
423 | proxy.emit('error', new got.ReadError(err, opts));
|
424 | });
|
425 |
|
426 | res.pipe(output);
|
427 |
|
428 | if (opts.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
|
429 | proxy.emit('error', new got.HTTPError(statusCode, res.statusMessage, res.headers, opts), null, res);
|
430 | return;
|
431 | }
|
432 |
|
433 | proxy.emit('response', res);
|
434 | });
|
435 |
|
436 | ee.on('error', proxy.emit.bind(proxy, 'error'));
|
437 | ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
|
438 | ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
|
439 | ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
|
440 |
|
441 | return proxy;
|
442 | }
|
443 |
|
444 | function normalizeArguments(url, opts) {
|
445 | if (!is.string(url) && !is.object(url)) {
|
446 | throw new TypeError(`Parameter \`url\` must be a string or object, not ${is(url)}`);
|
447 | } else if (is.string(url)) {
|
448 | url = url.replace(/^unix:/, 'http://$&');
|
449 |
|
450 | try {
|
451 | decodeURI(url);
|
452 | } catch (err) {
|
453 | throw new Error('Parameter `url` must contain valid UTF-8 character sequences');
|
454 | }
|
455 |
|
456 | url = urlParseLax(url);
|
457 | if (url.auth) {
|
458 | throw new Error('Basic authentication must be done with the `auth` option');
|
459 | }
|
460 | } else if (isURL.lenient(url)) {
|
461 | url = urlToOptions(url);
|
462 | }
|
463 |
|
464 | opts = Object.assign(
|
465 | {
|
466 | path: '',
|
467 | retries: 2,
|
468 | cache: false,
|
469 | decompress: true,
|
470 | useElectronNet: false,
|
471 | throwHttpErrors: true
|
472 | },
|
473 | url,
|
474 | {
|
475 | protocol: url.protocol || 'http:'
|
476 | },
|
477 | opts
|
478 | );
|
479 |
|
480 | const headers = lowercaseKeys(opts.headers);
|
481 | for (const key of Object.keys(headers)) {
|
482 | if (is.nullOrUndefined(headers[key])) {
|
483 | delete headers[key];
|
484 | }
|
485 | }
|
486 |
|
487 | opts.headers = Object.assign({
|
488 | 'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`,
|
489 | 'accept-encoding': 'gzip,deflate'
|
490 | }, headers);
|
491 |
|
492 | const query = opts.query;
|
493 |
|
494 | if (query) {
|
495 | if (!is.string(query)) {
|
496 | opts.query = querystring.stringify(query);
|
497 | }
|
498 |
|
499 | opts.path = `${opts.path.split('?')[0]}?${opts.query}`;
|
500 | delete opts.query;
|
501 | }
|
502 |
|
503 | if (opts.json && is.undefined(opts.headers.accept)) {
|
504 | opts.headers.accept = 'application/json';
|
505 | }
|
506 |
|
507 | const body = opts.body;
|
508 | if (is.nullOrUndefined(body)) {
|
509 | opts.method = (opts.method || 'GET').toUpperCase();
|
510 | } else {
|
511 | const headers = opts.headers;
|
512 | if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body) && !(opts.form || opts.json)) {
|
513 | throw new TypeError('The `body` option must be a stream.Readable, string, Buffer or plain Object');
|
514 | }
|
515 |
|
516 | const canBodyBeStringified = is.plainObject(body) || is.array(body);
|
517 | if ((opts.form || opts.json) && !canBodyBeStringified) {
|
518 | throw new TypeError('The `body` option must be a plain Object or Array when the `form` or `json` option is used');
|
519 | }
|
520 |
|
521 | if (isFormData(body)) {
|
522 |
|
523 | headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
|
524 | } else if (opts.form && canBodyBeStringified) {
|
525 | headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
|
526 | opts.body = querystring.stringify(body);
|
527 | } else if (opts.json && canBodyBeStringified) {
|
528 | headers['content-type'] = headers['content-type'] || 'application/json';
|
529 | opts.body = JSON.stringify(body);
|
530 | }
|
531 |
|
532 | if (is.undefined(headers['content-length']) && is.undefined(headers['transfer-encoding']) && !is.nodeStream(body)) {
|
533 | const length = is.string(opts.body) ? Buffer.byteLength(opts.body) : opts.body.length;
|
534 | headers['content-length'] = length;
|
535 | }
|
536 |
|
537 |
|
538 |
|
539 | if (is.buffer(body)) {
|
540 | opts.body = intoStream(body);
|
541 | opts.body._buffer = body;
|
542 | }
|
543 |
|
544 | opts.method = (opts.method || 'POST').toUpperCase();
|
545 | }
|
546 |
|
547 | if (opts.hostname === 'unix') {
|
548 | const matches = /(.+?):(.+)/.exec(opts.path);
|
549 |
|
550 | if (matches) {
|
551 | opts.socketPath = matches[1];
|
552 | opts.path = matches[2];
|
553 | opts.host = null;
|
554 | }
|
555 | }
|
556 |
|
557 | if (!is.function(opts.retries)) {
|
558 | const retries = opts.retries;
|
559 |
|
560 | opts.retries = (iter, err) => {
|
561 | if (iter > retries || !isRetryAllowed(err)) {
|
562 | return 0;
|
563 | }
|
564 |
|
565 | const noise = Math.random() * 100;
|
566 |
|
567 | return ((1 << iter) * 1000) + noise;
|
568 | };
|
569 | }
|
570 |
|
571 | if (is.undefined(opts.followRedirect)) {
|
572 | opts.followRedirect = true;
|
573 | }
|
574 |
|
575 | if (opts.timeout) {
|
576 | if (is.number(opts.timeout)) {
|
577 | opts.gotTimeout = {request: opts.timeout};
|
578 | } else {
|
579 | opts.gotTimeout = opts.timeout;
|
580 | }
|
581 | delete opts.timeout;
|
582 | }
|
583 |
|
584 | return opts;
|
585 | }
|
586 |
|
587 | function got(url, opts) {
|
588 | try {
|
589 | const normalizedArgs = normalizeArguments(url, opts);
|
590 |
|
591 | if (normalizedArgs.stream) {
|
592 | return asStream(normalizedArgs);
|
593 | }
|
594 |
|
595 | return asPromise(normalizedArgs);
|
596 | } catch (err) {
|
597 | return Promise.reject(err);
|
598 | }
|
599 | }
|
600 |
|
601 | got.stream = (url, opts) => asStream(normalizeArguments(url, opts));
|
602 |
|
603 | const methods = [
|
604 | 'get',
|
605 | 'post',
|
606 | 'put',
|
607 | 'patch',
|
608 | 'head',
|
609 | 'delete'
|
610 | ];
|
611 |
|
612 | for (const method of methods) {
|
613 | got[method] = (url, opts) => got(url, Object.assign({}, opts, {method}));
|
614 | got.stream[method] = (url, opts) => got.stream(url, Object.assign({}, opts, {method}));
|
615 | }
|
616 |
|
617 | Object.assign(got, errors);
|
618 |
|
619 | module.exports = got;
|