1 | var assert = require("assert");
|
2 | var _ = require("underscore");
|
3 | var seoaudit = require("../plugins/audit-plugin.js");
|
4 | var logger = require("../plugins/console-plugin.js");
|
5 | var cs = require("../plugins/console-plugin.js");
|
6 | var testSite = require("./website/start.js").site;
|
7 |
|
8 | var crawler = require("../index.js");
|
9 |
|
10 |
|
11 | describe('External Links', function() {
|
12 |
|
13 | it('Should not crawl external links & external domains by default', function(done) {
|
14 | var end = function(){
|
15 |
|
16 | assert(! audit.resources.get("http://www.nytimes.com"));
|
17 | assert(! audit.resources.get("http://www.nytimes.com/"));
|
18 | assert(audit.externalLinks.get("http://www.nytimes.com/") !== null);
|
19 | done();
|
20 |
|
21 | };
|
22 | crawler.init(null, end);
|
23 | var audit = new seoaudit.Plugin();
|
24 | crawler.registerPlugin(audit);
|
25 | crawler.queue({url : "http://localhost:9999/page2.html"});
|
26 |
|
27 | });
|
28 |
|
29 |
|
30 | it('Should crawl external links but not entire domains', function(done) {
|
31 | this.timeout(3000);
|
32 | var end = function(){
|
33 |
|
34 | assert(audit.resources.get("http://www.nytimes.com/"));
|
35 | assert(audit.externalLinks.get("http://www.nytimes.com/") !== null);
|
36 | done();
|
37 |
|
38 | };
|
39 | crawler.init({externalDomains : true, firstExternalLinkOnly : true}, end );
|
40 | var audit = new seoaudit.Plugin();
|
41 | crawler.registerPlugin(audit);
|
42 |
|
43 | crawler.queue({url : "http://localhost:9999/page2.html"});
|
44 |
|
45 | });
|
46 |
|
47 | it('Should not crawl domains that are in the black list', function(done) {
|
48 |
|
49 | var end = function(){
|
50 | assert(audit.resources.toArray().length === 0);
|
51 | assert(audit.errors.toArray()[0].error.code === "DOMAINBLACKLIST");
|
52 | done();
|
53 | };
|
54 | crawler.init(null, end);
|
55 | var audit = new seoaudit.Plugin();
|
56 | crawler.registerPlugin(audit);
|
57 | crawler.queue({url : "http://www.youtube.com"});
|
58 |
|
59 | });
|
60 |
|
61 |
|
62 | it('Plugins Should verify if the link is external or not', function(done) {
|
63 | var plugin = new TestPlugin();
|
64 | var end = function(){
|
65 | assert(plugin.isExternal);
|
66 | done();
|
67 | };
|
68 |
|
69 | crawler.init({externalDomains : true, firstExternalLinkOnly: true}, end);
|
70 | crawler.registerPlugin(plugin);
|
71 | crawler.queue({url : "http://localhost:9999/page12.html"});
|
72 |
|
73 | });
|
74 |
|
75 | });
|
76 |
|
77 |
|
78 |
|
79 | function TestPlugin() {
|
80 | this.name = "Test-Plugin";
|
81 | }
|
82 |
|
83 | TestPlugin.prototype.crawl = function (result,$, callback) {
|
84 |
|
85 | if (result.url === "http://cocoons.io/" && result.isExternal) {
|
86 | this.isExternal = true;
|
87 | }
|
88 | callback();
|
89 | };
|