1 |
|
2 |
|
3 | var utils = require('./utils'),
|
4 | fs = require('fs'),
|
5 | path = require('path'),
|
6 | forward = require('../lib/utils').forwardEvent,
|
7 | http = require('http'),
|
8 | https = require('https'),
|
9 | retire = require('./retire'),
|
10 | URL = require('url'),
|
11 | HttpsProxyAgent = require('https-proxy-agent');
|
12 |
|
13 | var emitter = require('events').EventEmitter;
|
14 |
|
15 |
|
16 | function loadJson(url, options) {
|
17 | var events = new emitter();
|
18 | options.log.info('Downloading ' + url + ' ...');
|
19 | var reqOptions = Object.assign({}, URL.parse(url), { method: 'GET' });
|
20 | if (options.proxy) {
|
21 | reqOptions.agent = new HttpsProxyAgent(options.proxy);
|
22 | }
|
23 | if (options.insecure) {
|
24 | reqOptions.rejectUnauthorized = false;
|
25 | }
|
26 | if (options.cacertbuf) {
|
27 | reqOptions.ca = [ options.cacertbuf ];
|
28 | }
|
29 | var req = (url.startsWith("http:") ? http : https).request(reqOptions, function (res) {
|
30 | if (res.statusCode != 200) return events.emit('stop', 'Error downloading: ' + url + ": HTTP " + res.statusCode + " " + res.statusText);
|
31 | var data = [];
|
32 | res.on('data', c => data.push(c));
|
33 | res.on('end', () => {
|
34 | var d = Buffer.concat(data).toString();
|
35 | d = options.process ? options.process(d) : d;
|
36 | events.emit('done', JSON.parse(d));
|
37 | });
|
38 | });
|
39 | req.on('error', e => events.emit('stop', 'Error downloading: ' + url + ": " + e.toString()));
|
40 | req.end();
|
41 | return events;
|
42 | }
|
43 |
|
44 | function loadJsonFromFile(file, options) {
|
45 | options.log.debug('Reading ' + file + ' ...');
|
46 | var events = new emitter();
|
47 | fs.readFile(file, { encoding : 'utf8'}, function(err, data) {
|
48 | if (err) { return events.emit('stop', err.toString()); }
|
49 | data = options.process ? options.process(data) : data;
|
50 | var obj = JSON.parse(data);
|
51 | events.emit('done', obj);
|
52 | });
|
53 | return events;
|
54 | }
|
55 |
|
56 | function loadFromCache(url, cachedir, options) {
|
57 | var cacheIndex = path.resolve(cachedir, 'index.json');
|
58 | if (!fs.existsSync(cachedir)) fs.mkdirSync(cachedir);
|
59 | var cache = fs.existsSync(cacheIndex) ? JSON.parse(fs.readFileSync(cacheIndex)) : {};
|
60 | var now = new Date().getTime();
|
61 | if (cache[url]) {
|
62 | if (now - cache[url].date < 60*60*1000) {
|
63 | options.log.info('Loading from cache: ' + url);
|
64 | return loadJsonFromFile(path.resolve(cachedir, cache[url].file), options);
|
65 | } else {
|
66 | if (fs.existsSync(path.resolve(cachedir, cache[url].date + '.json'))) {
|
67 | try {
|
68 | fs.unlinkSync(path.resolve(cachedir, cache[url].date + '.json'));
|
69 | } catch (error) {
|
70 | if (error.code !== 'ENOENT') {
|
71 | throw error;
|
72 | } else {
|
73 | console.warn("Could not delete cache. Ignore this error if you are running multiple retire.js in parallel");
|
74 | }
|
75 | }
|
76 | }
|
77 | }
|
78 | }
|
79 | var events = new emitter();
|
80 | loadJson(url, options).on('done', function(data) {
|
81 | cache[url] = { date : now, file : now + '.json' };
|
82 | fs.writeFileSync(path.resolve(cachedir, cache[url].file), JSON.stringify(data), { encoding : 'utf8' });
|
83 | fs.writeFileSync(cacheIndex, JSON.stringify(cache), { encoding : 'utf8' });
|
84 | events.emit('done', data);
|
85 | }).on('stop', forward(events, 'stop'));
|
86 | return events;
|
87 | }
|
88 |
|
89 | exports.asbowerrepo = function(jsRepo) {
|
90 | var result = {};
|
91 | Object.keys(jsRepo).map(function(k) {
|
92 | (jsRepo[k].bowername || [k]).map(function(b) {
|
93 | result[b] = result[b] || { vulnerabilities: [] };
|
94 | result[b].vulnerabilities = result[b].vulnerabilities.concat(jsRepo[k].vulnerabilities);
|
95 | });
|
96 | });
|
97 | return result;
|
98 | };
|
99 |
|
100 | exports.loadrepository = function(repoUrl, options) {
|
101 | options = utils.extend(options, { process : retire.replaceVersion });
|
102 | if (options.nocache) {
|
103 | return loadJson(repoUrl, options);
|
104 | }
|
105 | return loadFromCache(repoUrl, options.cachedir, options);
|
106 | };
|
107 |
|
108 | exports.loadrepositoryFromFile = function(filepath, options) {
|
109 | options = utils.extend(options, { process : retire.replaceVersion });
|
110 | return loadJsonFromFile(filepath, options);
|
111 | };
|