1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 | var ut = require('./utils');
|
7 |
|
8 | exports.Entry = Entry;
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 | function Entry () {
|
17 | this.userAgents = [];
|
18 | this.rules = [];
|
19 | }
|
20 |
|
21 |
|
22 |
|
23 |
|
24 |
|
25 |
|
26 |
|
27 | Entry.prototype.appliesTo = function(userAgent) {
|
28 |
|
29 | var userAgent = userAgent.split('/')[0].toLowerCase()
|
30 | , agent;
|
31 |
|
32 | for (var i = 0, len = this.userAgents.length; i < len; i++) {
|
33 | agent = this.userAgents[i].toLowerCase();
|
34 | if ( agent === '*' ||
|
35 | userAgent.indexOf( agent ) === 0
|
36 | ) {
|
37 | ut.d('* Entry.appliesTo, result:true, userAgent: '+userAgent+
|
38 | ', entryAgent: '+agent);
|
39 | return true;
|
40 | }
|
41 | };
|
42 | ut.d('* Entry.appliesTo, result:false');
|
43 | return false;
|
44 | };
|
45 |
|
46 |
|
47 |
|
48 |
|
49 |
|
50 |
|
51 |
|
52 |
|
53 |
|
54 | Entry.prototype.allowance = function(url) {
|
55 | ut.d('* Entry.allowance, url: '+url);
|
56 | for (var i = 0, len = this.rules.length, rule; i < len; i++) {
|
57 | rule = this.rules[i];
|
58 |
|
59 | if ( rule.appliesTo(url) ) {
|
60 | return rule.allowance;
|
61 | }
|
62 | };
|
63 |
|
64 | return true;
|
65 | };
|
66 |
|
67 | Entry.prototype.toString = function() {
|
68 | var res = [];
|
69 | for (var i = this.userAgents.length - 1; i >= 0; i--) {
|
70 | res.push('User-agent: '+this.userAgents[i]);
|
71 | };
|
72 | for (var i = this.rules.length - 1; i >= 0; i--) {
|
73 | res.push(this.rules[i]);
|
74 | };
|
75 | if(this.crawl_delay != null) {
|
76 | res.push('Crawl-Delay: ' + this.crawl_delay);
|
77 | }
|
78 | return "<Entry: " + res.join(', ') + ">";
|
79 | };
|