1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 | 'use strict';
|
7 |
|
8 | var Crawler = require('../lib/crawler');
|
9 | var expect = require('chai').expect;
|
10 | var nock = require('nock');
|
11 |
|
12 | var c;
|
13 | var tsArrs = [];
|
14 |
|
15 | describe('rateLimit tests', function () {
|
16 |
|
17 | before(function () {
|
18 | nock.cleanAll();
|
19 | });
|
20 |
|
21 | |
22 |
|
23 |
|
24 |
|
25 |
|
26 |
|
27 | beforeEach(function () {
|
28 | c = new Crawler({
|
29 | jquery: false,
|
30 | rateLimit: 300,
|
31 | callback: function (err, result, done) {
|
32 | expect(err).to.be.equal(null);
|
33 | expect(result.statusCode).to.equal(200);
|
34 | done();
|
35 | },
|
36 | });
|
37 | c.on('request', () => tsArrs.push(Date.now()));
|
38 | });
|
39 |
|
40 | afterEach(function () {
|
41 | c = {};
|
42 | tsArrs = [];
|
43 | });
|
44 |
|
45 | describe('Exceed rateLimit', function () {
|
46 | before(function () {
|
47 | nock('http://nockHost').get(uri => uri.indexOf('status') >= 0).times(2).delay(500).reply(200, 'Yes');
|
48 | });
|
49 |
|
50 | it('Interval of two requests should be no less than 500ms', function (testDone) {
|
51 | c.queue({ uri: 'http://nockHost/status/200' });
|
52 | c.queue({
|
53 | uri: 'http://nockHost/status/200',
|
54 | callback: function (err, result, done) {
|
55 | expect(err).to.be.equal(null);
|
56 | expect(result.statusCode).to.equal(200);
|
57 | done();
|
58 |
|
59 | expect(tsArrs.length).to.equal(2);
|
60 | expect(tsArrs[1] - tsArrs[0]).to.be.least(450);
|
61 |
|
62 | testDone();
|
63 | }
|
64 | });
|
65 | });
|
66 |
|
67 |
|
68 | });
|
69 |
|
70 | |
71 |
|
72 |
|
73 |
|
74 |
|
75 | describe('Abide rateLimit', function () {
|
76 |
|
77 |
|
78 | this.timeout(3000);
|
79 | before(function () {
|
80 | nock('http://nockHost').get(uri => uri.indexOf('status') >= 0).times(10).reply(200, 'Yes');
|
81 |
|
82 | });
|
83 |
|
84 | it('request speed should abide rateLimit', function (done) {
|
85 | for (var i = 0; i < 5; i++) {
|
86 | c.queue('http://nockHost/status/200');
|
87 | }
|
88 |
|
89 | c.on('drain', function () {
|
90 | expect(tsArrs.length).to.equal(5);
|
91 | for (var i = 1; i < tsArrs.length; i++) {
|
92 | var interval = tsArrs[i] - tsArrs[i - 1];
|
93 |
|
94 |
|
95 | var diff = Math.abs(interval - 300);
|
96 | expect(diff).to.be.most(30);
|
97 | }
|
98 |
|
99 | done();
|
100 | });
|
101 | });
|
102 |
|
103 | it('should be able to modify rateLimit', function (done) {
|
104 | c.setLimiterProperty('default', 'rateLimit', 500);
|
105 | for (var i = 0; i < 5; i++) {
|
106 | c.queue('http://nockHost/status/200');
|
107 | }
|
108 |
|
109 | c.on('drain', function () {
|
110 | expect(tsArrs.length).to.equal(5);
|
111 | for (var i = 1; i < tsArrs.length; i++) {
|
112 | var interval = tsArrs[i] - tsArrs[i - 1];
|
113 | var diff = Math.abs(interval - 500);
|
114 |
|
115 |
|
116 | expect(diff).to.be.at.most(50);
|
117 | }
|
118 |
|
119 | done();
|
120 | });
|
121 | });
|
122 |
|
123 | });
|
124 | }); |
\ | No newline at end of file |