1 | var assert = require("assert");
|
2 | var crawler = require("../index.js");
|
3 | var cs = require("../plugins/console-plugin.js");
|
4 | var testSite = require("./website-2/start.js").site;
|
5 | var proxyLoader = require("simple-proxies/lib/proxyfileloader");
|
6 | var log = require("crawler-ninja-logger").Logger;
|
7 |
|
8 |
|
9 |
|
10 | log.level("debug");
|
11 |
|
12 | var proxyListTest = null;
|
13 |
|
14 | describe('Memory leaks', function() {
|
15 |
|
16 | before(function(done) {
|
17 | var proxyFile = "./test/proxies-all.txt";
|
18 |
|
19 |
|
20 | var config = proxyLoader.config()
|
21 | .setProxyFile(proxyFile)
|
22 | .setCheckProxies(false)
|
23 | .setRemoveInvalidProxies(false);
|
24 |
|
25 | proxyLoader.loadProxyFile(config, function(error, proxyList) {
|
26 | if (error) {
|
27 | console.log(error);
|
28 |
|
29 | }
|
30 | else {
|
31 | proxyListTest = proxyList;
|
32 | done();
|
33 | }
|
34 |
|
35 | });
|
36 | });
|
37 | it.skip('should crawl a big site without memory leaks', function(done) {
|
38 | this.timeout(3000000);
|
39 | |
40 |
|
41 |
|
42 |
|
43 |
|
44 |
|
45 | var options = {
|
46 | skipDuplicates: true,
|
47 | externalDomains: false,
|
48 | scripts : false,
|
49 | links : false,
|
50 | images : false,
|
51 | maxConnections : 10
|
52 | }
|
53 | var consolePlugin = new cs.Plugin();
|
54 |
|
55 | crawler.init(options, done, null);
|
56 | crawler.registerPlugin(consolePlugin);
|
57 |
|
58 |
|
59 | crawler.queue({url : "http://www.rtbf.be"});
|
60 |
|
61 | });
|
62 |
|
63 | });
|