1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 | 'use strict';
|
8 |
|
9 | var Crawler = require('../lib/crawler');
|
10 | var expect = require('chai').expect;
|
11 | var nock = require('nock');
|
12 |
|
13 | var c;
|
14 | var tsArrs = [];
|
15 |
|
16 | describe('Limiter tests', function () {
|
17 | this.timeout(10000);
|
18 | before(function () {
|
19 | nock.cleanAll();
|
20 | });
|
21 | beforeEach(function () {
|
22 | nock('http://nockHost').get(uri => uri.indexOf('status') >= 0).times(5).reply(200, 'Yes');
|
23 |
|
24 | c = new Crawler({
|
25 | jquery: false,
|
26 | rateLimit: 500,
|
27 | callback: function (err, result, done) {
|
28 | expect(err).to.be.equal(null);
|
29 | expect(result.statusCode).to.equal(200);
|
30 | done();
|
31 | },
|
32 | });
|
33 | c.on('request', () => tsArrs.push(Date.now()));
|
34 | });
|
35 | afterEach(function () {
|
36 | c = {};
|
37 | tsArrs = [];
|
38 | });
|
39 |
|
40 | it('One limiter, tasks should execute one by one', function (done) {
|
41 | for (var i = 0; i < 5; i++) {
|
42 |
|
43 | c.queue({ uri: 'http://nockHost/status/200' });
|
44 | }
|
45 | c.on('drain', function () {
|
46 | expect(tsArrs.length).to.equal(5);
|
47 |
|
48 |
|
49 | expect(tsArrs[4] - tsArrs[0]).to.be.least(1950);
|
50 | expect(tsArrs[4] - tsArrs[0]).to.be.most(2050);
|
51 |
|
52 | done();
|
53 | });
|
54 | });
|
55 | it('Multiple limiters, tasks should execute in parallel', function (done) {
|
56 | for (var i = 0; i < 5; i++) {
|
57 | c.queue({ uri: 'http://nockHost/status/200', limiter: i });
|
58 | }
|
59 | c.on('drain', function () {
|
60 | expect(tsArrs.length).to.equal(5);
|
61 |
|
62 |
|
63 | expect(tsArrs[4] - tsArrs[0]).to.be.most(50);
|
64 |
|
65 | done();
|
66 | });
|
67 | });
|
68 | it('Multiple limiters are mutual independent', function (done) {
|
69 | for (var i = 0; i < 5; i++) {
|
70 | var limiter = i === 4 ? 'second' : 'default';
|
71 | c.queue({ uri: 'http://nockHost/status/200', limiter: limiter });
|
72 | }
|
73 | c.on('drain', function () {
|
74 | expect(tsArrs.length).to.equal(5);
|
75 |
|
76 |
|
77 | expect(tsArrs[4] - tsArrs[0]).to.be.least(1450);
|
78 | expect(tsArrs[4] - tsArrs[0]).to.be.most(1550);
|
79 |
|
80 | done();
|
81 | });
|
82 | });
|
83 | }); |
\ | No newline at end of file |