1 | var util = require ('util'),
|
2 | fs = require ('fs'),
|
3 | urlUtils = require ('url'),
|
4 | querystring = require ('querystring'),
|
5 | httpManager = require ('./http/model-manager'),
|
6 | tough = require ('tough-cookie'),
|
7 | path = require ('path');
|
8 |
|
9 | var HTTPClient, HTTPSClient, followRedirects;
|
10 |
|
11 | HTTPClient = require('http');
|
12 | HTTPSClient = require('https');
|
13 |
|
14 |
|
15 |
|
16 | var pipeProgress = function (config) {
|
17 | this.bytesToRead = 0;
|
18 | this.bytesRead = 0;
|
19 | this.bytesWritten = 0;
|
20 | this.lastLogged = 0;
|
21 | util.extend (this, config);
|
22 | };
|
23 |
|
24 | pipeProgress.prototype.watch = function () {
|
25 | var self = this;
|
26 | if (this.reader && this.readerWatch) {
|
27 | this.reader.on (this.readerWatch, function (chunk) {
|
28 | self.bytesRead += chunk.length;
|
29 | });
|
30 | } else if (this.writer && this.writerWatch) {
|
31 | this.writer.on (this.writerWatch, function (chunk) {
|
32 | self.bytesWritten += chunk.length;
|
33 | });
|
34 | }
|
35 |
|
36 | var readInterval = setInterval (function () {
|
37 | this.emit ('progress', this.bytesRead, this.bytesToRead);
|
38 | }.bind (this), 500);
|
39 |
|
40 | this.reader.on ('end', function () {
|
41 | clearInterval (readInterval);
|
42 | this.emit ('progress', this.bytesRead, this.bytesToRead);
|
43 | });
|
44 |
|
45 | this.reader.on ('error', function () {
|
46 | clearInterval (readInterval);
|
47 | });
|
48 |
|
49 | };
|
50 |
|
51 |
|
52 |
|
53 |
|
54 |
|
55 |
|
56 |
|
57 | var httpModel = module.exports = function (modelBase, optionalUrlParams) {
|
58 | this.modelBase = modelBase;
|
59 |
|
60 | this.params = {};
|
61 | this.extendParams(this.params, optionalUrlParams, modelBase.url);
|
62 |
|
63 | if (this.params.auth) {
|
64 | this.headers.Authorization = 'Basic ' +
|
65 | new Buffer(self.params.auth).toString('base64');
|
66 | }
|
67 |
|
68 | if (this.params.bodyData) {
|
69 | this.handleBodyData ();
|
70 | var method = this.params.method;
|
71 | this.params.method = (method && method.match (/POST|PUT/)) ? method : 'POST';
|
72 | this.bodyData = this.params.body;
|
73 |
|
74 | if (
|
75 | !this.params.headers ||
|
76 | !this.params.headers['content-length'] ||
|
77 | !this.params.headers['content-type']
|
78 | ) {
|
79 | console.error ('content type/length undefined');
|
80 | }
|
81 |
|
82 | delete this.params.bodyData;
|
83 |
|
84 | }
|
85 |
|
86 | this.redirectCount = 0;
|
87 | this.cookieJar = new tough.CookieJar (null, false);
|
88 |
|
89 | if (optionalUrlParams.proxy) {
|
90 | this.proxy = optionalUrlParams.proxy;
|
91 | }
|
92 |
|
93 | this.headers = {};
|
94 |
|
95 | if (this.params.headers) {
|
96 | try {
|
97 | util.extend(this.headers, this.params.headers);
|
98 | delete this.params.headers;
|
99 | } catch (e) {
|
100 | console.log ('headers is not correct');
|
101 | }
|
102 | }
|
103 | };
|
104 |
|
105 | httpModel.prototype.handleBodyData = function () {
|
106 |
|
107 | var bodyData = this.params.bodyData;
|
108 |
|
109 | var contentType = this.params.headers['content-type'],
|
110 | postType = Object.typeOf (bodyData);
|
111 |
|
112 |
|
113 | if (!contentType && postType == 'Object') {
|
114 | contentType = this.params.headers['content-type'] = 'application/x-www-form-urlencoded';
|
115 | } else if (!contentType) {
|
116 | contentType = 'undefined';
|
117 | }
|
118 |
|
119 | switch (contentType) {
|
120 | case 'application/x-www-form-urlencoded':
|
121 | this.params.body = querystring.stringify (bodyData);
|
122 | this.params.headers['content-length'] = this.params.body.length;
|
123 | break;
|
124 | case 'application/json':
|
125 | this.params.body = JSON.stringify (bodyData);
|
126 | this.params.headers['content-length'] = this.params.body.length;
|
127 | break;
|
128 | case 'multipart/mixed':
|
129 | case 'multipart/alternate':
|
130 | this.emitError ('multipart not yet implemented');
|
131 | return;
|
132 | break;
|
133 | case 'undefined':
|
134 | this.emitError ('you must define content type when submitting plain string as post data parameter');
|
135 | return;
|
136 | break;
|
137 | default:
|
138 | if (!this.params.headers['content-length']) {
|
139 | if (postType == 'String' || postType == 'Buffer') {
|
140 | this.params.headers['content-length'] = bodyData.length;
|
141 | } else {
|
142 | this.emitError ('you must define content-length when submitting plain string as post data parameter');
|
143 | return;
|
144 | }
|
145 | }
|
146 | break;
|
147 | }
|
148 |
|
149 | };
|
150 |
|
151 | util.extend (httpModel.prototype, {
|
152 | DefaultParams: {
|
153 | method: 'GET'
|
154 | },
|
155 |
|
156 | |
157 |
|
158 |
|
159 | UrlParamNames: [
|
160 | 'href',
|
161 |
|
162 |
|
163 |
|
164 | 'protocol',
|
165 |
|
166 |
|
167 |
|
168 | 'host',
|
169 |
|
170 |
|
171 |
|
172 |
|
173 | 'auth',
|
174 |
|
175 |
|
176 |
|
177 | 'hostname',
|
178 |
|
179 |
|
180 |
|
181 | 'port',
|
182 |
|
183 |
|
184 |
|
185 | 'pathname',
|
186 |
|
187 |
|
188 |
|
189 |
|
190 | 'search',
|
191 |
|
192 |
|
193 |
|
194 |
|
195 | 'path',
|
196 |
|
197 |
|
198 |
|
199 | 'query',
|
200 |
|
201 |
|
202 |
|
203 |
|
204 | 'hash'
|
205 |
|
206 |
|
207 | ],
|
208 |
|
209 | extendParams: function (params, configUrlObj, parsedUrlObj) {
|
210 | if (configUrlObj) {
|
211 | util.shallowMerge(params, configUrlObj, this.UrlParamNames);
|
212 | }
|
213 |
|
214 | if (parsedUrlObj) {
|
215 | util.shallowMerge(params, parsedUrlObj);
|
216 | }
|
217 |
|
218 |
|
219 | util.shallowMerge(params, this.DefaultParams);
|
220 |
|
221 | params.successCodes = configUrlObj.successCodes;
|
222 |
|
223 | params.bodyData = configUrlObj.bodyData;
|
224 |
|
225 |
|
226 |
|
227 | params.search = params.search || urlUtils.format({
|
228 | query: params.query
|
229 | });
|
230 |
|
231 | params.path = params.path || urlUtils.format({
|
232 | pathname: params.pathname,
|
233 | search: params.search
|
234 | });
|
235 |
|
236 | params.proxy = configUrlObj.proxy;
|
237 |
|
238 | params.href = params.href || urlUtils.format(params);
|
239 |
|
240 | params.port = params.port || ((this.params.protocol == 'https:') ? 443 : 80);
|
241 | params.protocol = params.protocol || this.params.protocol;
|
242 |
|
243 | return params;
|
244 | },
|
245 |
|
246 | fetch: function (target) {
|
247 | this.target = target;
|
248 |
|
249 | this.isStream = target.to instanceof fs.WriteStream;
|
250 |
|
251 | if (!this.isStream) target.to.data = new Buffer('');
|
252 |
|
253 | this.progress = new pipeProgress ({
|
254 | writer: target.to,
|
255 | emit: this.modelBase.emit.bind (this.modelBase)
|
256 | });
|
257 |
|
258 |
|
259 | global.httpModelManager.add(this, {
|
260 | url: this.params,
|
261 | headers: this.headers,
|
262 | bodyData: this.bodyData
|
263 | });
|
264 |
|
265 | return this.progress;
|
266 | },
|
267 | |
268 |
|
269 |
|
270 |
|
271 | addResultFields: function (result, meta) {
|
272 | if (this.res) {
|
273 | result.url = this.url;
|
274 | result.urlFileName = path.basename (this.url);
|
275 |
|
276 | result.code = this.res.statusCode || 500;
|
277 | if (result.stopReason === "timeout")
|
278 | result.code = 504;
|
279 | result.headers = (this.res.headers) ? this.res.headers : {};
|
280 | return;
|
281 | }
|
282 |
|
283 | if (meta) {
|
284 |
|
285 | result.code = meta.code;
|
286 | result.headers = meta.headers;
|
287 | result.url = meta.url;
|
288 | result.urlFileName = meta.urlFileName;
|
289 | return;
|
290 | }
|
291 |
|
292 | },
|
293 | isSuccessResponse: function check () {
|
294 | if (!this.res)
|
295 | return false;
|
296 | var statusCode = this.res.statusCode;
|
297 | if (this.params.successCodes) {
|
298 |
|
299 | var checkRe = new RegExp (this.params.successCodes.replace (/x/g, "\\d").replace (/,/g, "|"));
|
300 | if ((""+statusCode).match (checkRe)){
|
301 | return true;
|
302 | } else {
|
303 | return false;
|
304 | }
|
305 | } else if (statusCode == 200) {
|
306 | return true;
|
307 | }
|
308 | return false;
|
309 | },
|
310 | |
311 |
|
312 |
|
313 |
|
314 | run: function (params, headers, bodyData) {
|
315 | var self = this;
|
316 |
|
317 | var Client = (params.protocol === 'https:') ? HTTPSClient : HTTPClient;
|
318 |
|
319 | var requestParams = params;
|
320 |
|
321 | var requestUrl = params.href;
|
322 | this.url = requestUrl;
|
323 |
|
324 | if (this.proxy) {
|
325 | requestParams = urlUtils.parse (this.proxy);
|
326 | requestParams.path = params.href;
|
327 | requestParams.headers = {};
|
328 | for (var headerName in headers) {
|
329 | requestParams.headers[headerName] = headers[headerName];
|
330 | }
|
331 | requestParams.headers.host = params.host;
|
332 | }
|
333 |
|
334 | var req = self.req = Client.request (requestParams, function (res) {
|
335 |
|
336 | self.res = res;
|
337 | res.responseData = new Buffer (0);
|
338 |
|
339 | if (res.headers['set-cookie']) {
|
340 | if (res.headers['set-cookie'].constructor != Array) {
|
341 | res.headers['set-cookie'] = [res.headers['set-cookie']];
|
342 | }
|
343 | res.headers['set-cookie'].forEach (function (header) {
|
344 | self.cookieJar.setCookieSync (header, requestUrl);
|
345 |
|
346 | });
|
347 | }
|
348 |
|
349 | var redirected = self.isRedirected (requestUrl, res);
|
350 | if (redirected) {
|
351 | res.redirected = true;
|
352 | self.run (urlUtils.parse (redirected, true), {});
|
353 | this.redirectCount ++;
|
354 |
|
355 | req.end ();
|
356 | return;
|
357 | }
|
358 |
|
359 |
|
360 |
|
361 |
|
362 |
|
363 |
|
364 |
|
365 |
|
366 | util.extend (self.progress, {
|
367 | bytesToRead: res.headers['content-length'],
|
368 | reader: res,
|
369 | readerWatch: "data"
|
370 | });
|
371 |
|
372 | self.progress.watch ();
|
373 |
|
374 | if (self.isStream) {
|
375 | self.writeStream = self.target.to;
|
376 | res.pipe(self.writeStream);
|
377 | }
|
378 |
|
379 | res.on ('error', function (exception) {
|
380 | exception.scope = 'response';
|
381 | if (self.isStream) self.writeStream.end();
|
382 | self.modelBase.emit ('error', exception);
|
383 | });
|
384 |
|
385 |
|
386 | res.on ('data', function (chunk) {
|
387 | if (!self.isStream)
|
388 | res.responseData.length === 0
|
389 | ? res.responseData = chunk
|
390 | : res.responseData = Buffer.concat ([res.responseData, chunk]);
|
391 | self.modelBase.emit ('data', chunk);
|
392 | });
|
393 |
|
394 | res.on ('end', function () {
|
395 | if (!res.redirected) {
|
396 | self.target.to.data = res.responseData;
|
397 | delete res.responseData;
|
398 | self.modelBase.emit ('end');
|
399 | return;
|
400 | }
|
401 | self.modelBase.emit ('stop');
|
402 | res.jar = self.cookieJar;
|
403 | });
|
404 | });
|
405 |
|
406 | req.on ('error', function (exception) {
|
407 | self.res = self.res || {};
|
408 | exception.scope = 'request';
|
409 | if (self.stopReason)
|
410 | exception.stopReason = self.stopReason;
|
411 |
|
412 | self.modelBase.emit ('error', exception);
|
413 | });
|
414 |
|
415 | if (headers) {
|
416 | for (var key in headers) {
|
417 | req.setHeader(key, headers[key]);
|
418 | }
|
419 | }
|
420 |
|
421 | this.cookieJar.getCookiesSync (requestUrl).forEach (function (cookie) {
|
422 | req.setHeader ('cookie', cookie.cookieString());
|
423 | });
|
424 |
|
425 | if (bodyData) {
|
426 | req.write (bodyData);
|
427 | }
|
428 |
|
429 | req.end();
|
430 | },
|
431 |
|
432 | isRedirected: function (reqUrl, res) {
|
433 |
|
434 | if (!("" + res.statusCode).match (/^30[1,2,3,5,7]$/)) {
|
435 | return false;
|
436 | }
|
437 |
|
438 |
|
439 | if (!('location' in res.headers)) {
|
440 | return false;
|
441 | }
|
442 |
|
443 |
|
444 |
|
445 | res.on('data', function() {});
|
446 |
|
447 |
|
448 |
|
449 |
|
450 | var redirectUrl = urlUtils.resolve (reqUrl, "" + res.headers.location);
|
451 | return redirectUrl;
|
452 | },
|
453 |
|
454 | stop: function (reason) {
|
455 | if (reason)
|
456 | this.stopReason = reason;
|
457 | if (this.req) this.req.abort();
|
458 | if (this.res && this.res.destroy) this.res.destroy();
|
459 | }
|
460 | });
|