1 |
|
2 | 'use strict';
|
3 |
|
4 | const Crawler = require('../lib/crawler');
|
5 | const expect = require('chai').expect;
|
6 | const jsdom = require('jsdom');
|
7 |
|
8 |
|
9 | const nock = require('nock');
|
10 |
|
11 | describe('Errors', function() {
|
12 |
|
13 | before(function() {
|
14 | nock.cleanAll();
|
15 | nock('http://test.crawler.com').get('/delay/1').delay(1000).reply(200, 'ok').persist();
|
16 | nock('http://test.crawler.com').get('/status/400').reply(400, 'Bad Request').persist();
|
17 | nock('http://test.crawler.com').get('/status/401').reply(401, 'Unauthorized').persist();
|
18 | nock('http://test.crawler.com').get('/status/403').reply(403, 'Forbidden').persist();
|
19 | nock('http://test.crawler.com').get('/status/404').reply(404, 'Not Found').persist();
|
20 | nock('http://test.crawler.com').get('/status/500').reply(500, 'Internal Error').persist();
|
21 | nock('http://test.crawler.com').get('/status/204').reply(204, '').persist();
|
22 | });
|
23 |
|
24 | describe('timeout', function() {
|
25 | const crawler = new Crawler({
|
26 | timeout: 500,
|
27 | retryTimeout: 1000,
|
28 | retries: 2,
|
29 | jquery: false
|
30 | });
|
31 |
|
32 | it('should retry after timeout', function(finishTest) {
|
33 | this.timeout(10000);
|
34 | let options = {
|
35 | uri: 'http://test.crawler.com/delay/1',
|
36 | callback: (error, response, done) => {
|
37 | expect(error).to.exist;
|
38 | expect(response.options.retries).to.equal(0);
|
39 | done();
|
40 | finishTest();
|
41 | }
|
42 | };
|
43 | crawler.queue(options);
|
44 | expect(options.retries).to.equal(2);
|
45 | });
|
46 |
|
47 | it('should return a timeout error after ~2sec', function(finishTest) {
|
48 | this.timeout(10000);
|
49 | crawler.queue({
|
50 | uri: 'http://test.crawler.com/delay/1',
|
51 | callback: (error, response, done) => {
|
52 | expect(error).to.exist;
|
53 | expect(error.code === 'ETIMEDOUT' || error.code === 'ESOCKETTIMEDOUT').to.be.true;
|
54 | done();
|
55 | finishTest();
|
56 | }
|
57 | });
|
58 | });
|
59 | });
|
60 |
|
61 | describe('error status code', function() {
|
62 | const crawler = new Crawler({ jQuery : false });
|
63 |
|
64 | it('should not return an error on status code 400 (Bad Request)', function(finishTest) {
|
65 | crawler.queue({
|
66 | uri: 'http://test.crawler.com/status/400',
|
67 | callback: (error, response, done) => {
|
68 | expect(error).to.be.null;
|
69 | expect(response.statusCode).to.equal(400);
|
70 | done();
|
71 | finishTest();
|
72 | }
|
73 | });
|
74 | });
|
75 |
|
76 | it('should not return an error on status code 401 (Unauthorized)', function(finishTest) {
|
77 | crawler.queue({
|
78 | uri: 'http://test.crawler.com/status/401',
|
79 | callback: (error, response, done) => {
|
80 | expect(error).to.be.null;
|
81 | expect(response.statusCode).to.equal(401);
|
82 | done();
|
83 | finishTest();
|
84 | }
|
85 | });
|
86 | });
|
87 |
|
88 | it('should not return an error on status code 403 (Forbidden)', function(finishTest) {
|
89 | crawler.queue({
|
90 | uri: 'http://test.crawler.com/status/403',
|
91 | callback: (error, response, done) => {
|
92 | expect(error).to.be.null;
|
93 | expect(response.statusCode).to.equal(403);
|
94 | done();
|
95 | finishTest();
|
96 | }
|
97 | });
|
98 | });
|
99 |
|
100 | it('should not return an error on a 404', function(finishTest) {
|
101 | crawler.queue({
|
102 | uri : 'http://test.crawler.com/status/404',
|
103 | callback : (error, response, done) => {
|
104 | expect(error).to.be.null;
|
105 | expect(response.statusCode).to.equal(404);
|
106 | done();
|
107 | finishTest();
|
108 | }
|
109 | });
|
110 | });
|
111 |
|
112 | it('should not return an error on a 500', function(finishTest) {
|
113 | crawler.queue({
|
114 | uri : 'http://test.crawler.com/status/500',
|
115 | callback : (error, response, done) => {
|
116 | expect(error).to.be.null;
|
117 | expect(response.statusCode).to.equal(500);
|
118 | done();
|
119 | finishTest();
|
120 | }
|
121 | });
|
122 | });
|
123 |
|
124 | it('should not failed on empty response', function(finishTest) {
|
125 | crawler.queue({
|
126 | uri : 'http://test.crawler.com/status/204',
|
127 | callback : (error, response, done) => {
|
128 | expect(error).to.be.null;
|
129 | done();
|
130 | finishTest();
|
131 | }
|
132 | });
|
133 | });
|
134 |
|
135 | it('should not failed on a malformed html if jquery is false', function(finishTest) {
|
136 | crawler.queue({
|
137 | html : '<html><p>hello <div>dude</p></html>',
|
138 | callback : (error, response, done) => {
|
139 | expect(error).to.be.null;
|
140 | expect(response).not.to.be.null;
|
141 | done();
|
142 | finishTest();
|
143 | }
|
144 | });
|
145 | });
|
146 |
|
147 | it('should not return an error on a malformed html if jQuery is jsdom', function(finishTest) {
|
148 | crawler.queue({
|
149 | html : '<html><p>hello <div>dude</p></html>',
|
150 | jQuery : jsdom,
|
151 | callback : (error, response, done) => {
|
152 | expect(error).to.be.null;
|
153 | expect(response).not.to.be.undefined;
|
154 | done();
|
155 | finishTest();
|
156 | }
|
157 | });
|
158 | });
|
159 | });
|
160 | }); |
\ | No newline at end of file |