1 | ;
|
2 | Object.defineProperty(exports, "__esModule", { value: true });
|
3 | exports.getGot = void 0;
|
4 | const url_1 = require("url");
|
5 | const js_lib_1 = require("@naturalcycles/js-lib");
|
6 | const got_1 = require("got");
|
7 | const __1 = require("..");
|
8 | /**
|
9 | * Returns instance of Got with "reasonable defaults":
|
10 | *
|
11 | * 1. Error handler hook that prints helpful errors.
|
12 | * 2. Hooks that log start/end of request (optional, false by default).
|
13 | * 3. Reasonable defaults(tm), e.g non-infinite Timeout
|
14 | * 4. Preserves error stack traces (!) (experimental!)
|
15 | */
|
16 | function getGot(opt = {}) {
|
17 | opt.logger || (opt.logger = console);
|
18 | if (opt.debug) {
|
19 | opt.logStart = opt.logFinished = opt.logResponse = opt.logRequest = true;
|
20 | }
|
21 | return got_1.default.extend({
|
22 | // Most-important is to set to anything non-empty (so, requests don't "hang" by default).
|
23 | // Should be long enough to handle for slow responses from scaled cloud APIs in times of spikes
|
24 | // Ideally should be LESS than default Request timeout in backend-lib (so, it has a chance to error
|
25 | // before server times out with 503).
|
26 | //
|
27 | // UPD 2021-11-27
|
28 | // There are 2 types/strategies for requests:
|
29 | // 1. Optimized to get result no matter what. E.g in Cron jobs, where otherwise there'll be a job failure
|
30 | // 2. Part of the Backend request, where we better retry quickly and fail on timeout before Backend aborts it with "503 Request timeout"
|
31 | //
|
32 | // Here it's hard to set the default timeout right for both use-cases.
|
33 | // So, if it's important, you should override it according to your use-cases:
|
34 | // - set it longer for Type 1 (e.g 120 seconds)
|
35 | // - set it shorter for Type 2 (e.g 10/20 seconds)
|
36 | // Please beware of default Retry strategy of Got:
|
37 | // by default it will retry 2 times (after first try)
|
38 | // First delay between tries will be ~1 second, then ~2 seconds
|
39 | // Each retry it'll wait up to `timeout` (so, up to 60 seconds by default).
|
40 | // So, for 3 tries it multiplies your timeout by 3 (+3 seconds between the tries).
|
41 | // So, e.g 60 seconds timeout with 2 retries becomes up to 183 seconds.
|
42 | // Which definitely doesn't fit into default "RequestTimeout"
|
43 | timeout: 60000,
|
44 | ...opt,
|
45 | handlers: [
|
46 | (options, next) => {
|
47 | options.context = {
|
48 | ...options.context,
|
49 | started: Date.now(),
|
50 | // This is to preserve original stack trace
|
51 | // https://github.com/sindresorhus/got/blob/main/documentation/async-stack-traces.md
|
52 | err: new Error('RequestError'),
|
53 | };
|
54 | return next(options);
|
55 | },
|
56 | ],
|
57 | hooks: {
|
58 | ...opt.hooks,
|
59 | beforeError: [
|
60 | ...(opt.hooks?.beforeError || []),
|
61 | // User hooks go BEFORE
|
62 | gotErrorHook(opt),
|
63 | ],
|
64 | beforeRequest: [
|
65 | gotBeforeRequestHook(opt),
|
66 | // User hooks go AFTER
|
67 | ...(opt.hooks?.beforeRequest || []),
|
68 | ],
|
69 | beforeRetry: [
|
70 | gotBeforeRetryHook(opt),
|
71 | // User hooks go AFTER
|
72 | ...(opt.hooks?.beforeRetry || []),
|
73 | ],
|
74 | afterResponse: [
|
75 | ...(opt.hooks?.afterResponse || []),
|
76 | // User hooks go BEFORE
|
77 | gotAfterResponseHook(opt),
|
78 | ],
|
79 | },
|
80 | });
|
81 | }
|
82 | exports.getGot = getGot;
|
83 | /**
|
84 | * Without this hook (default behaviour):
|
85 | *
|
86 | * HTTPError: Response code 422 (Unprocessable Entity)
|
87 | * at EventEmitter.<anonymous> (.../node_modules/got/dist/source/as-promise.js:118:31)
|
88 | * at processTicksAndRejections (internal/process/task_queues.js:97:5) {
|
89 | * name: 'HTTPError'
|
90 | *
|
91 | *
|
92 | * With this hook:
|
93 | *
|
94 | * HTTPError 422 GET http://a.com/err?q=1 in 8 ms
|
95 | * {
|
96 | * message: 'Reference already exists',
|
97 | * documentation_url: 'https://developer.github.com/v3/git/refs/#create-a-reference'
|
98 | * }
|
99 | *
|
100 | * Features:
|
101 | * 1. Includes original method and URL (including e.g searchParams) in the error message.
|
102 | * 2. Includes response.body in the error message (limited length).
|
103 | * 3. Auto-detects and parses JSON response body (limited length).
|
104 | * 4. Includes time spent (gotBeforeRequestHook must also be enabled).
|
105 | * UPD: excluded now to allow automatic Sentry error grouping
|
106 | */
|
107 | function gotErrorHook(opt = {}) {
|
108 | const { maxResponseLength = 10000 } = opt;
|
109 | return err => {
|
110 | const statusCode = err.response?.statusCode || 0;
|
111 | const { method, url, prefixUrl } = err.options;
|
112 | const shortUrl = getShortUrl(opt, url, prefixUrl);
|
113 | const { started, retryCount } = (err.request?.options.context || {});
|
114 | const body = err.response?.body
|
115 | ? (0, __1.inspectAny)(err.response.body, {
|
116 | maxLen: maxResponseLength,
|
117 | colors: false,
|
118 | })
|
119 | : err.message;
|
120 | // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
|
121 | opt.logger.log([
|
122 | ' <<',
|
123 | statusCode,
|
124 | method,
|
125 | shortUrl,
|
126 | retryCount && `(retry ${retryCount})`,
|
127 | 'error',
|
128 | started && 'in ' + (0, js_lib_1._since)(started),
|
129 | ]
|
130 | .filter(Boolean)
|
131 | .join(' '));
|
132 | // timings are not part of err.message to allow automatic error grouping in Sentry
|
133 | // Colors are not used, because there's high chance that this Error will be propagated all the way to the Frontend
|
134 | err.message = [[statusCode, method, shortUrl].filter(Boolean).join(' '), body]
|
135 | .filter(Boolean)
|
136 | .join('\n');
|
137 | const stack = err.options.context?.err?.stack;
|
138 | if (stack) {
|
139 | const originalStack = err.stack.split('\n');
|
140 | let originalStackIndex = originalStack.findIndex(line => line.includes(' at '));
|
141 | if (originalStackIndex === -1)
|
142 | originalStackIndex = originalStack.length - 1;
|
143 | // Skipping first line as it has RequestError: ...
|
144 | // Skipping second line as it's known to be from e.g at got_1.default.extend.handlers
|
145 | const syntheticStack = stack.split('\n').slice(2);
|
146 | let firstNonNodeModulesIndex = syntheticStack.findIndex(line => !line.includes('node_modules'));
|
147 | if (firstNonNodeModulesIndex === -1)
|
148 | firstNonNodeModulesIndex = 0;
|
149 | err.stack = [
|
150 | // First lines of original error
|
151 | ...originalStack.slice(0, originalStackIndex),
|
152 | // Other lines from "Synthetic error"
|
153 | ...syntheticStack.slice(firstNonNodeModulesIndex),
|
154 | ].join('\n');
|
155 | // err.stack += '\n --' + stack.replace('Error: RequestError', '')
|
156 | }
|
157 | return err;
|
158 | };
|
159 | }
|
160 | function gotBeforeRequestHook(opt) {
|
161 | return options => {
|
162 | if (opt.logStart) {
|
163 | const { retryCount } = options.context;
|
164 | const shortUrl = getShortUrl(opt, options.url, options.prefixUrl);
|
165 | opt.logger.log([' >>', options.method, shortUrl, retryCount && `(retry ${retryCount})`].join(' '));
|
166 | }
|
167 | if (opt.logRequest) {
|
168 | const body = options.json || options.body;
|
169 | if (body) {
|
170 | opt.logger.log(body);
|
171 | }
|
172 | }
|
173 | };
|
174 | }
|
175 | // Here we log always, because it's similar to ErrorHook - we always log errors
|
176 | // Because Retries are always result of some Error
|
177 | function gotBeforeRetryHook(opt) {
|
178 | const { maxResponseLength = 10000 } = opt;
|
179 | return (options, err, retryCount) => {
|
180 | // opt.logger!.log('beforeRetry', retryCount)
|
181 | const statusCode = err?.response?.statusCode || 0;
|
182 | if (statusCode && statusCode < 300) {
|
183 | // todo: possibly remove the log message completely in the future
|
184 | // opt.logger!.log(
|
185 | // `skipping got.beforeRetry hook as statusCode is ${statusCode}, err.msg is ${err?.message}`,
|
186 | // )
|
187 | return;
|
188 | }
|
189 | const { method, url, prefixUrl } = options;
|
190 | const shortUrl = getShortUrl(opt, url, prefixUrl);
|
191 | const { started } = options.context;
|
192 | Object.assign(options.context, { retryCount });
|
193 | const body = err?.response?.body
|
194 | ? (0, __1.inspectAny)(err.response.body, {
|
195 | maxLen: maxResponseLength,
|
196 | colors: false,
|
197 | })
|
198 | : err?.message;
|
199 | // We don't include Response/Body/Message in the log, because it's included in the Error thrown from here
|
200 | opt.logger.warn([
|
201 | [
|
202 | ' <<',
|
203 | statusCode,
|
204 | method,
|
205 | shortUrl,
|
206 | retryCount && retryCount > 1 ? `(retry ${retryCount - 1})` : '(first try)',
|
207 | 'error',
|
208 | started && 'in ' + (0, js_lib_1._since)(started),
|
209 | ]
|
210 | .filter(Boolean)
|
211 | .join(' '),
|
212 | body,
|
213 | ]
|
214 | .filter(Boolean)
|
215 | .join('\n'));
|
216 | };
|
217 | }
|
218 | // AfterResponseHook is never called on Error
|
219 | // So, coloredHttpCode(resp.statusCode) is probably useless
|
220 | function gotAfterResponseHook(opt = {}) {
|
221 | return resp => {
|
222 | const success = resp.statusCode >= 200 && resp.statusCode < 400;
|
223 | // Errors are not logged here, as they're logged by gotErrorHook
|
224 | if (opt.logFinished && success) {
|
225 | const { started, retryCount } = resp.request.options.context;
|
226 | const { url, prefixUrl, method } = resp.request.options;
|
227 | const shortUrl = getShortUrl(opt, url, prefixUrl);
|
228 | opt.logger.log([
|
229 | ' <<',
|
230 | resp.statusCode,
|
231 | method,
|
232 | shortUrl,
|
233 | retryCount && `(retry ${retryCount - 1})`,
|
234 | started && 'in ' + (0, js_lib_1._since)(started),
|
235 | ]
|
236 | .filter(Boolean)
|
237 | .join(' '));
|
238 | // console.log(`afterResp! ${resp.request.options.method} ${resp.url}`, { context: resp.request.options.context })
|
239 | }
|
240 | // Error responses are not logged, cause they're included in Error message already
|
241 | if (opt.logResponse && success) {
|
242 | opt.logger.log((0, __1.inspectAny)(resp.body, { maxLen: opt.maxResponseLength }));
|
243 | }
|
244 | return resp;
|
245 | };
|
246 | }
|
247 | function getShortUrl(opt, url, prefixUrl) {
|
248 | if (url.password) {
|
249 | url = new url_1.URL(url.toString()); // prevent original url mutation
|
250 | url.password = '[redacted]';
|
251 | }
|
252 | let shortUrl = url.toString();
|
253 | if (opt.logWithSearchParams === false) {
|
254 | shortUrl = shortUrl.split('?')[0];
|
255 | }
|
256 | if (opt.logWithPrefixUrl === false && prefixUrl && shortUrl.startsWith(prefixUrl)) {
|
257 | shortUrl = shortUrl.slice(prefixUrl.length);
|
258 | }
|
259 | return shortUrl;
|
260 | }
|