1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 | global.logger_component = 'engine';
|
13 |
|
14 | var
|
15 | logger = require('joola.io.logger'),
|
16 |
|
17 | _datasources = require('./lib/objects/datasources'),
|
18 | _datatables = require('./lib/objects/datatables'),
|
19 | connector = require('./lib/connectors/connector'),
|
20 | caching = require('./lib/caching/manager.js'),
|
21 |
|
22 | _ = global._ = require('underscore'),
|
23 | ce = global.ce = require('cloneextend'),
|
24 |
|
25 | express = require('express'),
|
26 | http = require('http'),
|
27 | https = require('https'),
|
28 | path = require('path'),
|
29 | nconf = require('nconf'),
|
30 | async = require('async');
|
31 |
|
32 | require('nconf-http');
|
33 | require('pkginfo')(module, 'version');
|
34 | require('date-utils');
|
35 |
|
36 | var joola = {};
|
37 | global.joola = joola;
|
38 |
|
39 | joola.state = {
|
40 | status: 'init',
|
41 | running: false
|
42 | };
|
43 |
|
44 | joola.logger = logger;
|
45 |
|
46 | joola.config = nconf;
|
47 | joola.config.server = null;
|
48 | joola.config.auth = null;
|
49 | joola.config.integration = null;
|
50 | joola.config.content = null;
|
51 |
|
52 | var
|
53 | httpServer,
|
54 | httpsServer,
|
55 | app = global.app = express();
|
56 |
|
57 | joola.config.add('base-config', { type: 'file', file: joola.config.get('confurl') || './config/config.json' });
|
58 |
|
59 | var fetchConfig = function (name, file, callback) {
|
60 | joola.logger.debug('Fetching configuration from: ' + joola.config.get('config:url') + file);
|
61 | joola.config.add(name, { type: 'http', url: joola.config.get('config:url') + file, callback: function (err) {
|
62 | if (err) {
|
63 |
|
64 | joola.logger.warn('Failed to fetch config, fallback to local FS: ' + joola.config.get('config:url') + file);
|
65 | joola.config.add(name, { type: 'file', file: joola.config.get('conf') || './config/' + file + '.json' });
|
66 | }
|
67 |
|
68 | if (!joola.config.get(name + ':version'))
|
69 | return callback(new Error('Failed to load configuration file: ' + './config/' + file + '.json'));
|
70 |
|
71 | return callback(null);
|
72 | }});
|
73 | };
|
74 |
|
75 | var loadConfig = function (callback) {
|
76 | joola.state.status = 'loadConfig';
|
77 | joola.config.argv()
|
78 | .env();
|
79 |
|
80 |
|
81 |
|
82 | fetchConfig('server', 'joola.io.engine.server', function (err) {
|
83 | if (err)
|
84 | return callback(err);
|
85 | joola.config.server = joola.config.get('server');
|
86 |
|
87 | fetchConfig('auth', 'joola.io.engine.auth', function (err) {
|
88 | if (err)
|
89 | return callback(err);
|
90 | joola.config.auth = joola.config.get('auth');
|
91 |
|
92 | fetchConfig('integration', 'joola.io.engine.integration', function (err) {
|
93 | if (err)
|
94 | return callback(err);
|
95 | joola.config.integration = joola.config.get('integration');
|
96 |
|
97 | fetchConfig('content', 'joola.io.engine.content', function (err) {
|
98 | if (err)
|
99 | return callback(err);
|
100 | joola.config.content = joola.config.get('content');
|
101 | return callback(null);
|
102 | });
|
103 | });
|
104 | });
|
105 | });
|
106 | };
|
107 |
|
108 | var setupCache = function (callback) {
|
109 | var port, host, dbid, options;
|
110 | var errorReported = false;
|
111 |
|
112 | host = joola.config.get('server:redis:host');
|
113 | port = joola.config.get('server:redis:port');
|
114 | dbid = joola.config.get('server:redis:DB');
|
115 |
|
116 | if (!dbid) {
|
117 | joola.config.set('server:redis:DB', 0);
|
118 | }
|
119 |
|
120 | var handleRedisError = function (err) {
|
121 | return joola.logger.error('Redis error: ' + err);
|
122 | };
|
123 |
|
124 | require('./lib/shared/redis').redis(joola, function (err) {
|
125 | if (err) {
|
126 | if (errorReported) {
|
127 | return handleRedisError(err);
|
128 | }
|
129 | else {
|
130 | errorReported = true;
|
131 | return callback(err);
|
132 | }
|
133 | }
|
134 | require('./lib/shared/cache').cache(joola, function (err) {
|
135 | if (err) {
|
136 | if (errorReported)
|
137 | return handleRedisError(err);
|
138 | else {
|
139 | errorReported = true;
|
140 | return callback(err);
|
141 | }
|
142 | }
|
143 |
|
144 | return callback(null);
|
145 | });
|
146 | });
|
147 | };
|
148 |
|
149 | var flushRedis = function (callback) {
|
150 | joola.redis.flushdb(function (err) {
|
151 | if (err)
|
152 | return callback(err);
|
153 |
|
154 | joola.logger.warn('Redis cache flushed');
|
155 |
|
156 | callback(null);
|
157 | });
|
158 | };
|
159 |
|
160 | var processIntegration = function (callback) {
|
161 | joola.state.status = 'processIntegration';
|
162 | var datasources = joola.config.integration.datasources;
|
163 | datasources.forEach(function (ds) {
|
164 | if (!ds.id)
|
165 | throw 'JSON Config Validation Error: Source ' + ds.name + ' has no id';
|
166 |
|
167 | _.each(ds.datatables, function (dt) {
|
168 | if (!dt.id)
|
169 | throw 'JSON Config Validation Error: Table ' + dt.name + ' has no id';
|
170 |
|
171 | dt.datasourceid = ds.id;
|
172 | });
|
173 | });
|
174 |
|
175 | return callback(null);
|
176 | };
|
177 |
|
178 | var verifyDatasources = function (callback) {
|
179 | var datasources = joola.config.integration.datasources;
|
180 |
|
181 | var checks = [];
|
182 | var allChecksPassed = true;
|
183 | datasources.forEach(function (ds) {
|
184 | var check = function (callback) {
|
185 | joola.logger.debug('Checking data source [' + ds.id + ']...');
|
186 | _datasources.validate(ds.id, function (err, validated) {
|
187 | if (err) {
|
188 | joola.logger.warn('Failed to validate data source: ' + ds.id);
|
189 | allChecksPassed = false;
|
190 | }
|
191 | else if (!validated) {
|
192 | joola.logger.warn('Failed to validate data source: ' + ds.id);
|
193 | allChecksPassed = false;
|
194 | }
|
195 |
|
196 | if (joola.config.get('flushcache')) {
|
197 | ds.datatables.forEach(function (dt) {
|
198 | dt.datasource = ds;
|
199 | _datatables.flushCache(dt, function () {
|
200 |
|
201 | });
|
202 | });
|
203 | return callback();
|
204 | }
|
205 | else
|
206 | return callback();
|
207 | });
|
208 | };
|
209 | checks.push(check);
|
210 | });
|
211 |
|
212 | async.parallel(checks, function () {
|
213 | if (allChecksPassed)
|
214 | return callback(null);
|
215 | else
|
216 | return callback(new Error('Some data source checks failed, check log for more details.'));
|
217 | });
|
218 | };
|
219 |
|
220 | var preFlight = function (callback) {
|
221 | joola.state.status = 'preFlight';
|
222 | loadConfig(function (err) {
|
223 | if (err)
|
224 | return callback(err);
|
225 |
|
226 | joola.logger.info('Loaded server config version ' + joola.config.get('server:version'));
|
227 | joola.logger.info('Loaded auth config version ' + joola.config.get('auth:version'));
|
228 | joola.logger.info('Loaded integration config version ' + joola.config.get('integration:version'));
|
229 | joola.logger.info('Loaded content config version ' + joola.config.get('content:version'));
|
230 |
|
231 | joola.logger.setLevel(joola.config.server.logLevel);
|
232 |
|
233 | setupCache(function (err) {
|
234 | if (err) {
|
235 | return callback(err);
|
236 | }
|
237 |
|
238 | processIntegration(function (err) {
|
239 | if (err)
|
240 | return callback(err);
|
241 |
|
242 | verifyDatasources(function (err) {
|
243 | if (err)
|
244 | return callback(err);
|
245 |
|
246 | return callback(null);
|
247 | });
|
248 | });
|
249 | });
|
250 | });
|
251 | };
|
252 |
|
253 | var start = function (callback) {
|
254 | joola.state.status = 'starting';
|
255 | joola.state.running = false;
|
256 |
|
257 | setupApplication(function (err) {
|
258 | if (err)
|
259 | return callback(err);
|
260 |
|
261 | setupRoutes(function (err) {
|
262 | if (err)
|
263 | return callback(err);
|
264 |
|
265 | setupControlPort(function () {
|
266 | joola.logger.info('Control port running on port ' + joola.config.get('server:controlPort:port'));
|
267 |
|
268 | startHTTP(function () {
|
269 | joola.logger.debug('HTTP running');
|
270 |
|
271 | if (joola.config.get('server:secure') === true) {
|
272 | startHTTPS(function () {
|
273 | joola.logger.debug('HTTPS running');
|
274 |
|
275 | return callback(null);
|
276 | });
|
277 | }
|
278 | else
|
279 | return callback(null);
|
280 | });
|
281 | });
|
282 | });
|
283 | });
|
284 | };
|
285 |
|
286 | var setupApplication = function (callback) {
|
287 | try {
|
288 | var winstonStream = {
|
289 | write: function (message, encoding) {
|
290 |
|
291 | }
|
292 | };
|
293 | app.use(express.logger((global.test ? function (req, res) {
|
294 | } : {stream: winstonStream})));
|
295 |
|
296 |
|
297 | app.set('views', __dirname + '/views');
|
298 | app.set('view engine', 'jade');
|
299 | app.disable('x-powered-by');
|
300 | app.use(express.compress());
|
301 | app.use(express.favicon('public/assets/ico/favicon.ico'));
|
302 | app.use(express.bodyParser());
|
303 | app.use(express.methodOverride());
|
304 | app.use(express.cookieParser());
|
305 | app.use(express.session({
|
306 | secret: 'what-should-be-the-secret?',
|
307 | maxAge: new Date(Date.now() + 3600000),
|
308 | expires: new Date(Date.now() + 3600000)
|
309 | }));
|
310 | app.use(require('./lib/middleware/headers')());
|
311 | app.use(require('joola.io.auth')(joola.config.get('server:auth')));
|
312 |
|
313 | return callback(null);
|
314 | }
|
315 | catch (err) {
|
316 | joola.logger.error('setupApplication: ' + err);
|
317 | return callback(err);
|
318 | }
|
319 | };
|
320 |
|
321 | var setupRoutes = function (callback) {
|
322 | try {
|
323 | var
|
324 | index = require('./routes/index');
|
325 |
|
326 | app.configure(function () {
|
327 | app.use(function (req, res, next) {
|
328 | res.setHeader('Access-Control-Allow-Origin', '*');
|
329 | res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
|
330 | res.setHeader('Access-Control-Allow-Headers', 'joola-token, Content-Type, origin');
|
331 | res.setHeader('Access-Control-Max-Age', '86400');
|
332 | res.setHeader('X-Powered-By', 'joola.io');
|
333 |
|
334 | return next();
|
335 | });
|
336 | });
|
337 |
|
338 | app.get('/', index.index);
|
339 | app.get('/:resource', index.route);
|
340 | app.get('/:resource/:action', index.route);
|
341 | app.use(express.static(path.join(__dirname, 'public')));
|
342 | app.use(app.router);
|
343 |
|
344 |
|
345 |
|
346 |
|
347 | return callback(null);
|
348 | }
|
349 | catch (err) {
|
350 | joola.logger.error('setupRoutes: ' + err);
|
351 | return callback(err);
|
352 | }
|
353 | };
|
354 |
|
355 |
|
356 | var startHTTP = function (callback) {
|
357 | var result = {};
|
358 | try {
|
359 | var _httpServer = http.createServer(app).listen(joola.config.get('server:port'),function (err) {
|
360 | if (err) {
|
361 | result.status = 'Failed: ' + ex.message;
|
362 | return callback(result);
|
363 | }
|
364 | joola.state.status = 'running';
|
365 | joola.state.running = true;
|
366 |
|
367 | joola.logger.info('Joola Analytics HTTP server listening on port ' + joola.config.get('server:port'));
|
368 | httpServer = _httpServer;
|
369 | return callback(result);
|
370 | }).on('error',function (ex) {
|
371 | result.status = 'Failed: ' + ex.message;
|
372 | return callback(result);
|
373 | }).on('close', function () {
|
374 |
|
375 | joola.state.status = 'stopped';
|
376 | joola.state.running = false;
|
377 |
|
378 | joola.logger.warn('Joola Analytics HTTP server listening on port ' + (joola.config.get('server:port')).toString() + ' received a CLOSE command.');
|
379 | });
|
380 | }
|
381 | catch (ex) {
|
382 | result.status = 'Failed: ' + ex.message;
|
383 | return callback(result);
|
384 | }
|
385 | return null;
|
386 | };
|
387 |
|
388 | var startHTTPS = function (callback) {
|
389 | var result = {};
|
390 | try {
|
391 | var secureOptions = {
|
392 | key: fs.readFileSync(joola.config.get('server:keyFile')),
|
393 | cert: fs.readFileSync(joola.config.get('server:certFile'))
|
394 | };
|
395 | var _httpsServer = https.createServer(secureOptions, app).listen(joola.config.get('server:securePort'),function (err) {
|
396 | if (err) {
|
397 | result.status = 'Failed: ' + ex.message;
|
398 | return callback(result);
|
399 | }
|
400 | joola.logger.info('Joola Analytics HTTPS server listening on port ' + joola.config.get('server:securePort'));
|
401 | result.status = 'Success';
|
402 | httpsServer = _httpsServer;
|
403 | return callback(result);
|
404 | }).on('error',function (ex) {
|
405 | result.status = 'Failed: ' + ex.message;
|
406 | return callback(result);
|
407 | }).on('close', function () {
|
408 | joola.logger.warn('Joola Analytics HTTPS server listening on port ' + joola.config.get('server:securePort').toString() + ' received a CLOSE command.');
|
409 | });
|
410 | }
|
411 | catch (ex) {
|
412 | result.status = 'Failed: ' + ex.message;
|
413 | return callback(result);
|
414 | }
|
415 | return null;
|
416 | };
|
417 |
|
418 |
|
419 | var setupControlPort = function (callback) {
|
420 | var cp = require('node-controlport');
|
421 | var cp_endpoints = [];
|
422 |
|
423 | cp_endpoints.push({
|
424 | endpoint: 'status',
|
425 | exec: function (callback) {
|
426 | return callback({status: joola.state.status, pid: process.pid});
|
427 | }
|
428 | });
|
429 |
|
430 | cp_endpoints.push({
|
431 | endpoint: 'start',
|
432 | exec: function (callback) {
|
433 | if (joola.config.get('server:secure') === true) {
|
434 | startHTTP(function () {
|
435 | startHTTPS(callback);
|
436 | });
|
437 | }
|
438 | else {
|
439 | startHTTP(callback);
|
440 | }
|
441 | }
|
442 | }
|
443 | );
|
444 |
|
445 | cp_endpoints.push({
|
446 | endpoint: 'stop',
|
447 | exec: function (callback) {
|
448 | var result = {};
|
449 | result.status = 'Success';
|
450 | try {
|
451 | httpServer.close();
|
452 | if (joola.config.get('server:secure') === true)
|
453 | httpsServer.close();
|
454 |
|
455 | process.exit(0);
|
456 | }
|
457 | catch (ex) {
|
458 | console.log(ex);
|
459 | result.status = 'Failed: ' + ex.message;
|
460 | return callback(result);
|
461 | }
|
462 | return callback(result);
|
463 | }
|
464 | });
|
465 |
|
466 | cp.start(joola.config.get('server:controlPort:port'), cp_endpoints, callback);
|
467 | };
|
468 |
|
469 | var fetchEndDates = function (callback) {
|
470 | joola.logger.debug('Fetching end dates...');
|
471 | var datasources = _datasources.list(true);
|
472 |
|
473 |
|
474 | var async_calls = [];
|
475 |
|
476 | datasources.forEach(function (ds) {
|
477 | if (ds.enddate) {
|
478 | if (ds.enddate.type == 'query') {
|
479 | var query = connector.createQuery();
|
480 | query.datasource = ds;
|
481 | query.sql = ds.enddate.query;
|
482 |
|
483 | joola.logger.debug('Fetching end dates for datasource - ' + ds.id);
|
484 |
|
485 | var call = function (callback) {
|
486 | connector.executeQuery(query, function (query, rows, fields, error) {
|
487 | if (error)
|
488 | throw error;
|
489 |
|
490 | var row = rows.rows[0];
|
491 | var col = rows.fields[0].name;
|
492 | var timestamp;
|
493 | try {
|
494 | timestamp = new Date(row[col]);
|
495 | }
|
496 | catch (ex) {
|
497 |
|
498 | timestamp = new Date();
|
499 | }
|
500 | timestamp.setMilliseconds(timestamp.getMilliseconds() - 1);
|
501 |
|
502 | joola.logger.debug('... [' + ds.id + '] ' + timestamp.format('yyyy-mm-dd hh:nn:ss'));
|
503 | ds.enddate.value = timestamp;
|
504 | return callback();
|
505 | });
|
506 | };
|
507 | async_calls.push(call);
|
508 | }
|
509 | }
|
510 | });
|
511 |
|
512 | fork(async_calls, function () {
|
513 | joola.logger.debug('...Datasource end dates fetched.');
|
514 | |
515 |
|
516 |
|
517 |
|
518 |
|
519 |
|
520 | return callback();
|
521 | });
|
522 | };
|
523 |
|
524 | var buildInitialCache = function (callback) {
|
525 | var range;
|
526 |
|
527 |
|
528 | var found = false;
|
529 | var datatables = _datatables.list();
|
530 | var calls = [];
|
531 |
|
532 | var errorReported = false;
|
533 |
|
534 | var cacheTable = function (args, callback) {
|
535 | var query = args.query;
|
536 | var dt = args.dt;
|
537 | query.enddate = ce.clone(args.enddate);
|
538 | query.startdate = ce.clone(args.startdate);
|
539 | caching.cacheTable(dt, ce.clone(query), function (err) {
|
540 | return callback(err);
|
541 | });
|
542 | };
|
543 |
|
544 | datatables.forEach(function (dt) {
|
545 | if (dt.caching) {
|
546 | if (dt.caching.method == 'persist' || dt.caching.method == 'forward') {
|
547 | found = true;
|
548 | joola.logger.debug('Found [' + dt.id + '] for cache management [' + dt.caching.method + ']...');
|
549 | var step = dt.caching.step;
|
550 |
|
551 |
|
552 | var ds = _datasources.get(dt.datasourceid);
|
553 | var query = connector.createQuery();
|
554 |
|
555 | dt.datasource = ds;
|
556 | dt.query = _datatables.basequery(dt);
|
557 |
|
558 | var basequery = dt.query;
|
559 | query.sql = basequery.sql;
|
560 | query.datasource = ds;
|
561 |
|
562 | var enddate = ce.clone(ds.enddate.value);
|
563 | var startdate = ce.clone(enddate);
|
564 | startdate.setMilliseconds(startdate.getMilliseconds() + 1);
|
565 | startdate.addDays(-1 * parseInt((range ? range : dt.caching.baseline), 10));
|
566 |
|
567 | var _baseline_start = ce.clone(startdate);
|
568 |
|
569 | query.enddate = enddate;
|
570 | query.startdate = startdate;
|
571 |
|
572 | var firstRun = true;
|
573 |
|
574 | if (_baseline_start.dateDiff(enddate) > step) {
|
575 | while (startdate >= _baseline_start) {
|
576 | if (firstRun) {
|
577 | startdate = ce.clone(enddate);
|
578 | startdate.setMilliseconds(startdate.getMilliseconds() + 1);
|
579 | firstRun = false;
|
580 | }
|
581 | startdate.addDays(-1 * parseInt((range ? range : dt.caching.step), 10));
|
582 | calls.push({query: query, dt: dt, startdate: ce.clone(startdate), enddate: ce.clone(enddate)});
|
583 |
|
584 | enddate = ce.clone(startdate);
|
585 | enddate.setMilliseconds(enddate.getMilliseconds() - 1);
|
586 | }
|
587 | }
|
588 | else {
|
589 | calls.push({query: query, dt: dt, startdate: ce.clone(startdate), enddate: ce.clone(enddate)});
|
590 | }
|
591 | }
|
592 | }
|
593 | });
|
594 | async.mapSeries(calls, cacheTable, function (err, results) {
|
595 | if (err)
|
596 | return callback(err);
|
597 |
|
598 | joola.logger.info('Cache pre-load completed.');
|
599 |
|
600 | |
601 |
|
602 |
|
603 |
|
604 |
|
605 | return callback(null);
|
606 | });
|
607 |
|
608 | if (!found)
|
609 | return callback(null);
|
610 | };
|
611 |
|
612 | var setupEagerCache = function (callback) {
|
613 | var tables = _datatables.list();
|
614 | _.each(tables, function (dt) {
|
615 | dt.datasource = _datasources.get(dt.datasourceid);
|
616 | if (dt.caching && dt.caching.eager && dt.caching.eager.interval && dt.caching.eager.interval > 0) {
|
617 |
|
618 | joola.logger.debug('Setting up eagerCache for [' + dt.id + '] task with interval: ' + dt.caching.eager.interval);
|
619 | var tFunction = function () {
|
620 | caching.eagerCache(dt, function () {
|
621 | joola.logger.debug('Re-setting eagerCache for [' + dt.id + '] task with interval: ' + dt.caching.eager.interval);
|
622 | setTimeout(tFunction, dt.caching.eager.interval);
|
623 | });
|
624 | };
|
625 |
|
626 | setTimeout(tFunction, dt.caching.eager.interval);
|
627 | }
|
628 | });
|
629 |
|
630 | return callback();
|
631 | };
|
632 |
|
633 |
|
634 | joola.logger.info('Starting joola.io Engine, version ' + module.exports.version + '...');
|
635 |
|
636 | try {
|
637 | preFlight(function (err) {
|
638 | if (err) {
|
639 | console.log('ERR1');
|
640 | throw err;
|
641 | }
|
642 |
|
643 | joola.logger.info('Preflight checks done, starting engine...');
|
644 | try {
|
645 | start(function (err) {
|
646 | if (err)
|
647 | throw err;
|
648 |
|
649 | joola.logger.info('joole.io Engine started!');
|
650 |
|
651 | if (joola.config.get('flushcache')) {
|
652 | flushRedis(function (err) {
|
653 | if (err)
|
654 | throw err;
|
655 | });
|
656 | }
|
657 |
|
658 | fetchEndDates(function (err) {
|
659 | if (err)
|
660 | throw err;
|
661 | buildInitialCache(function (err) {
|
662 | if (err)
|
663 | throw err;
|
664 | setupEagerCache(function (err) {
|
665 | if (err)
|
666 | throw err;
|
667 | });
|
668 | });
|
669 | logger.info('Cache ready!');
|
670 | });
|
671 | });
|
672 | }
|
673 | catch (ex) {
|
674 | joola.logger.error('FATAL EXCEPTION (start): ' + ex.message + '\n' + ex.stack, null, function () {
|
675 | process.exit(1);
|
676 | });
|
677 | }
|
678 | });
|
679 | }
|
680 | catch (ex) {
|
681 | joola.logger.error('FATAL EXCEPTION (preflight): ' + ex.message + '\n' + ex.stack, null, function () {
|
682 | process.exit(1);
|
683 | });
|
684 | }
|
685 |
|
686 | process.on('uncaughtException', function (exception) {
|
687 |
|
688 | console.log('FATAL EXCEPTION: ' + exception.message);
|
689 | console.log(exception.stack);
|
690 |
|
691 | joola.logger.error('FATAL EXCEPTION: ' + exception.message + '\n' + exception.stack, null, function () {
|
692 | process.exit(1);
|
693 | });
|
694 | }); |
\ | No newline at end of file |