1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 | var semver = require('semver');
|
18 | var cluster = require('cluster');
|
19 | var numCPUs = require('os').cpus() ? require('os').cpus().length : 1;
|
20 | var path = require('path');
|
21 | var EventEmitter2 = require('eventemitter2').EventEmitter2;
|
22 | var fs = require('fs');
|
23 | var vizion = require('vizion');
|
24 | var debug = require('debug')('pm2:god');
|
25 | var Utility = require('./Utility');
|
26 | var cst = require('../constants.js');
|
27 | var timesLimit = require('async/timesLimit');
|
28 |
|
29 |
|
30 |
|
31 |
|
32 | cluster.setupMaster({
|
33 | windowsHide: true,
|
34 | exec : path.resolve(path.dirname(module.filename), 'ProcessContainer.js')
|
35 | });
|
36 |
|
37 |
|
38 |
|
39 |
|
40 | var God = module.exports = {
|
41 | next_id : 0,
|
42 | clusters_db : {},
|
43 | started_at : Date.now(),
|
44 | bus : new EventEmitter2({
|
45 | wildcard: true,
|
46 | delimiter: ':',
|
47 | maxListeners: 1000
|
48 | })
|
49 | };
|
50 |
|
51 | Utility.overrideConsole(God.bus);
|
52 |
|
53 |
|
54 |
|
55 |
|
56 | require('./Event.js')(God);
|
57 | require('./God/Methods.js')(God);
|
58 | require('./God/ForkMode.js')(God);
|
59 | require('./God/ClusterMode.js')(God);
|
60 | require('./God/Reload')(God);
|
61 | require('./God/ActionMethods')(God);
|
62 | require('./Watcher')(God);
|
63 |
|
64 |
|
65 |
|
66 |
|
67 |
|
68 |
|
69 |
|
70 |
|
71 |
|
72 | God.executeApp = function executeApp(env, cb) {
|
73 | var env_copy = Utility.clone(env);
|
74 |
|
75 | Utility.extend(env_copy, env_copy.env);
|
76 |
|
77 | env_copy['status'] = cst.LAUNCHING_STATUS;
|
78 | env_copy['pm_uptime'] = Date.now();
|
79 | env_copy['axm_actions'] = [];
|
80 | env_copy['axm_monitor'] = {};
|
81 | env_copy['axm_options'] = {};
|
82 | env_copy['axm_dynamic'] = {};
|
83 | env_copy['vizion_running'] =
|
84 | env_copy['vizion_running'] !== undefined ? env_copy['vizion_running'] : false;
|
85 |
|
86 | if (!env_copy.created_at)
|
87 | env_copy['created_at'] = Date.now();
|
88 |
|
89 | |
90 |
|
91 |
|
92 |
|
93 |
|
94 |
|
95 |
|
96 | if (env_copy['pm_id'] === undefined) {
|
97 | env_copy['pm_id'] = God.getNewId();
|
98 | env_copy['restart_time'] = 0;
|
99 | env_copy['unstable_restarts'] = 0;
|
100 |
|
101 |
|
102 | env_copy.pm_pid_path = env_copy.pm_pid_path.replace(/-[0-9]+\.pid$|\.pid$/g, '-' + env_copy['pm_id'] + '.pid');
|
103 |
|
104 |
|
105 | if (!env_copy['merge_logs']) {
|
106 | ['', '_out', '_err'].forEach(function(k){
|
107 | var key = 'pm' + k + '_log_path';
|
108 | env_copy[key] && (env_copy[key] = env_copy[key].replace(/-[0-9]+\.log$|\.log$/g, '-' + env_copy['pm_id'] + '.log'));
|
109 | });
|
110 | }
|
111 |
|
112 |
|
113 | if (env_copy['watch']) {
|
114 | God.watch.enable(env_copy);
|
115 | }
|
116 | }
|
117 |
|
118 | |
119 |
|
120 |
|
121 |
|
122 | var workAround = function(worker) {
|
123 | var listeners = null;
|
124 |
|
125 | listeners = worker.process.listeners('exit')[0];
|
126 | var exit = listeners[Object.keys(listeners)[0]];
|
127 |
|
128 | listeners = worker.process.listeners('disconnect')[0];
|
129 | var disconnect = listeners[Object.keys(listeners)[0]];
|
130 |
|
131 | worker.process.removeListener('exit', exit);
|
132 | worker.process.once('exit', function(exitCode, signalCode) {
|
133 |
|
134 |
|
135 | if (worker.state != 'disconnected')
|
136 | disconnect();
|
137 |
|
138 | exit(exitCode, signalCode);
|
139 | });
|
140 | };
|
141 |
|
142 |
|
143 | var readyCb = function ready(proc) {
|
144 | if (proc.pm2_env.vizion !== false && proc.pm2_env.vizion !== "false")
|
145 | God.finalizeProcedure(proc);
|
146 | else
|
147 | God.notify('online', proc);
|
148 |
|
149 | proc.pm2_env.status = cst.ONLINE_STATUS;
|
150 | console.log(`App [${proc.pm2_env.name}:${proc.pm2_env.pm_id}] online`);
|
151 | if (cb) cb(null, proc);
|
152 | }
|
153 |
|
154 | if (env_copy.exec_mode === 'cluster_mode') {
|
155 | |
156 |
|
157 |
|
158 | God.nodeApp(env_copy, function nodeApp(err, clu) {
|
159 | if (cb && err) return cb(err);
|
160 | if (err) return false;
|
161 |
|
162 | var old_env = God.clusters_db[clu.pm2_env.pm_id];
|
163 |
|
164 | if (old_env) {
|
165 | old_env = null;
|
166 | God.clusters_db[clu.pm2_env.pm_id] = null;
|
167 | }
|
168 |
|
169 | God.clusters_db[clu.pm2_env.pm_id] = clu;
|
170 |
|
171 | if (semver.lt(process.version, '7.0.0') === true) {
|
172 |
|
173 | workAround(clu);
|
174 | }
|
175 |
|
176 | clu.once('error', function(err) {
|
177 | console.error(err.stack || err);
|
178 | clu.pm2_env.status = cst.ERRORED_STATUS;
|
179 | try {
|
180 | clu.destroy && clu.destroy();
|
181 | }
|
182 | catch (e) {
|
183 | console.error(e.stack || e);
|
184 | God.handleExit(clu, cst.ERROR_EXIT);
|
185 | }
|
186 | });
|
187 |
|
188 | clu.once('disconnect', function() {
|
189 | console.log('App name:%s id:%s disconnected', clu.pm2_env.name, clu.pm2_env.pm_id);
|
190 | });
|
191 |
|
192 | clu.once('exit', function cluExit(code, signal) {
|
193 | God.handleExit(clu, code || 0, signal || 'SIGINT');
|
194 | });
|
195 |
|
196 | return clu.once('online', function () {
|
197 | if (!clu.pm2_env.wait_ready)
|
198 | return readyCb(clu);
|
199 |
|
200 |
|
201 | var ready_timeout = setTimeout(function() {
|
202 | God.bus.removeListener('process:msg', listener)
|
203 | return readyCb(clu)
|
204 | }, clu.pm2_env.listen_timeout || cst.GRACEFUL_LISTEN_TIMEOUT);
|
205 |
|
206 | var listener = function (packet) {
|
207 | if (packet.raw === 'ready' &&
|
208 | packet.process.name === clu.pm2_env.name &&
|
209 | packet.process.pm_id === clu.pm2_env.pm_id) {
|
210 | clearTimeout(ready_timeout);
|
211 | God.bus.removeListener('process:msg', listener)
|
212 | return readyCb(clu)
|
213 | }
|
214 | }
|
215 |
|
216 | God.bus.on('process:msg', listener);
|
217 | });
|
218 | });
|
219 | }
|
220 | else {
|
221 | |
222 |
|
223 |
|
224 | God.forkMode(env_copy, function forkMode(err, clu) {
|
225 | if (cb && err) return cb(err);
|
226 | if (err) return false;
|
227 |
|
228 | var old_env = God.clusters_db[clu.pm2_env.pm_id];
|
229 | if (old_env) old_env = null;
|
230 |
|
231 | God.clusters_db[env_copy.pm_id] = clu;
|
232 |
|
233 | clu.once('error', function cluError(err) {
|
234 | console.error(err.stack || err);
|
235 | clu.pm2_env.status = cst.ERRORED_STATUS;
|
236 | try {
|
237 | clu.kill && clu.kill();
|
238 | }
|
239 | catch (e) {
|
240 | console.error(e.stack || e);
|
241 | God.handleExit(clu, cst.ERROR_EXIT);
|
242 | }
|
243 | });
|
244 |
|
245 | clu.once('exit', function cluClose(code, signal) {
|
246 | if (clu.connected === true)
|
247 | clu.disconnect && clu.disconnect();
|
248 | clu._reloadLogs = null;
|
249 | return God.handleExit(clu, code || 0, signal);
|
250 | });
|
251 |
|
252 | if (!clu.pm2_env.wait_ready)
|
253 | return readyCb(clu);
|
254 |
|
255 |
|
256 | var ready_timeout = setTimeout(function() {
|
257 | God.bus.removeListener('process:msg', listener)
|
258 | return readyCb(clu)
|
259 | }, clu.pm2_env.listen_timeout || cst.GRACEFUL_LISTEN_TIMEOUT);
|
260 |
|
261 | var listener = function (packet) {
|
262 | if (packet.raw === 'ready' &&
|
263 | packet.process.name === clu.pm2_env.name &&
|
264 | packet.process.pm_id === clu.pm2_env.pm_id) {
|
265 | clearTimeout(ready_timeout);
|
266 | God.bus.removeListener('process:msg', listener)
|
267 | return readyCb(clu)
|
268 | }
|
269 | }
|
270 | God.bus.on('process:msg', listener);
|
271 | });
|
272 | }
|
273 | return false;
|
274 | };
|
275 |
|
276 |
|
277 |
|
278 |
|
279 |
|
280 |
|
281 |
|
282 |
|
283 | God.handleExit = function handleExit(clu, exit_code, kill_signal) {
|
284 | console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] exited with code [${exit_code}] via signal [${kill_signal || 'SIGINT'}]`)
|
285 |
|
286 | var proc = this.clusters_db[clu.pm2_env.pm_id];
|
287 |
|
288 | if (!proc) {
|
289 | console.error('Process undefined ? with process id ', clu.pm2_env.pm_id);
|
290 | return false;
|
291 | }
|
292 |
|
293 | var stopping = (proc.pm2_env.status == cst.STOPPING_STATUS
|
294 | || proc.pm2_env.status == cst.STOPPED_STATUS
|
295 | || proc.pm2_env.status == cst.ERRORED_STATUS) || (proc.pm2_env.autorestart === false ||
|
296 | proc.pm2_env.autorestart === "false");
|
297 |
|
298 | var overlimit = false;
|
299 |
|
300 | if (stopping) proc.process.pid = 0;
|
301 |
|
302 |
|
303 | if (proc.pm2_env.axm_actions) proc.pm2_env.axm_actions = [];
|
304 | if (proc.pm2_env.axm_monitor) proc.pm2_env.axm_monitor = {};
|
305 |
|
306 | if (proc.pm2_env.status != cst.ERRORED_STATUS &&
|
307 | proc.pm2_env.status != cst.STOPPING_STATUS)
|
308 | proc.pm2_env.status = cst.STOPPED_STATUS;
|
309 |
|
310 | if (proc.pm2_env.pm_id.toString().indexOf('_old_') !== 0) {
|
311 | try {
|
312 | fs.unlinkSync(proc.pm2_env.pm_pid_path);
|
313 | } catch (e) {
|
314 | debug('Error when unlinking pid file', e);
|
315 | }
|
316 | }
|
317 |
|
318 | |
319 |
|
320 |
|
321 |
|
322 |
|
323 |
|
324 | var min_uptime = typeof(proc.pm2_env.min_uptime) !== 'undefined' ? proc.pm2_env.min_uptime : 1000;
|
325 | var max_restarts = typeof(proc.pm2_env.max_restarts) !== 'undefined' ? proc.pm2_env.max_restarts : 16;
|
326 |
|
327 | if ((Date.now() - proc.pm2_env.created_at) < (min_uptime * max_restarts)) {
|
328 | if ((Date.now() - proc.pm2_env.pm_uptime) < min_uptime) {
|
329 |
|
330 | proc.pm2_env.unstable_restarts += 1;
|
331 | }
|
332 | }
|
333 |
|
334 |
|
335 | if (proc.pm2_env.unstable_restarts >= max_restarts) {
|
336 |
|
337 |
|
338 |
|
339 | proc.pm2_env.status = cst.ERRORED_STATUS;
|
340 | proc.process.pid = 0;
|
341 |
|
342 | console.log('Script %s had too many unstable restarts (%d). Stopped. %j',
|
343 | proc.pm2_env.pm_exec_path,
|
344 | proc.pm2_env.unstable_restarts,
|
345 | proc.pm2_env.status);
|
346 |
|
347 | God.notify('restart overlimit', proc);
|
348 |
|
349 | proc.pm2_env.unstable_restarts = 0;
|
350 | proc.pm2_env.created_at = null;
|
351 | overlimit = true;
|
352 | }
|
353 |
|
354 | if (typeof(exit_code) !== 'undefined') proc.pm2_env.exit_code = exit_code;
|
355 |
|
356 | God.notify('exit', proc);
|
357 |
|
358 | if (God.pm2_being_killed) {
|
359 |
|
360 | return false;
|
361 | }
|
362 |
|
363 | var restart_delay = 0;
|
364 |
|
365 | if (proc.pm2_env.restart_delay !== undefined &&
|
366 | !isNaN(parseInt(proc.pm2_env.restart_delay))) {
|
367 | proc.pm2_env.status = cst.WAITING_RESTART;
|
368 | restart_delay = parseInt(proc.pm2_env.restart_delay);
|
369 | }
|
370 |
|
371 | if (proc.pm2_env.exp_backoff_restart_delay !== undefined &&
|
372 | !isNaN(parseInt(proc.pm2_env.exp_backoff_restart_delay))) {
|
373 | proc.pm2_env.status = cst.WAITING_RESTART;
|
374 | if (!proc.pm2_env.prev_restart_delay) {
|
375 | proc.pm2_env.prev_restart_delay = proc.pm2_env.exp_backoff_restart_delay
|
376 | restart_delay = proc.pm2_env.exp_backoff_restart_delay
|
377 | }
|
378 | else {
|
379 | proc.pm2_env.prev_restart_delay = Math.floor(Math.min(15000, proc.pm2_env.prev_restart_delay * 1.5))
|
380 | restart_delay = proc.pm2_env.prev_restart_delay
|
381 | }
|
382 | console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] will restart in ${restart_delay}ms`)
|
383 | }
|
384 |
|
385 | if (!stopping && !overlimit) {
|
386 |
|
387 | Object.defineProperty(proc.pm2_env, 'restart_task', {configurable: true, writable: true});
|
388 | proc.pm2_env.restart_task = setTimeout(function() {
|
389 | proc.pm2_env.restart_time += 1;
|
390 | God.executeApp(proc.pm2_env);
|
391 | }, restart_delay);
|
392 | }
|
393 |
|
394 | return false;
|
395 | };
|
396 |
|
397 |
|
398 |
|
399 |
|
400 | God.prepare = function prepare (env, cb) {
|
401 |
|
402 | env.env.unique_id = Utility.generateUUID()
|
403 |
|
404 |
|
405 | if (typeof env.instances === 'undefined') {
|
406 | env.vizion_running = false;
|
407 | if (env.env && env.env.vizion_running) env.env.vizion_running = false;
|
408 |
|
409 | return God.executeApp(env, function (err, clu) {
|
410 | if (err) return cb(err);
|
411 | God.notify('start', clu, true);
|
412 | return cb(null, [ Utility.clone(clu) ]);
|
413 | });
|
414 | }
|
415 |
|
416 |
|
417 | env.instances = parseInt(env.instances);
|
418 | if (env.instances === 0) {
|
419 | env.instances = numCPUs;
|
420 | } else if (env.instances < 0) {
|
421 | env.instances += numCPUs;
|
422 | }
|
423 | if (env.instances <= 0) {
|
424 | env.instances = 1;
|
425 | }
|
426 |
|
427 | timesLimit(env.instances, 1, function (n, next) {
|
428 | env.vizion_running = false;
|
429 | if (env.env && env.env.vizion_running) {
|
430 | env.env.vizion_running = false;
|
431 | }
|
432 |
|
433 | God.injectVariables(env, function inject (err, _env) {
|
434 | if (err) return next(err);
|
435 | return God.executeApp(Utility.clone(_env), function (err, clu) {
|
436 | if (err) return next(err);
|
437 | God.notify('start', clu, true);
|
438 |
|
439 |
|
440 | return next(null, Utility.clone(clu));
|
441 | });
|
442 | });
|
443 | }, cb);
|
444 | };
|
445 |
|
446 |
|
447 |
|
448 |
|
449 |
|
450 |
|
451 | God.finalizeProcedure = function finalizeProcedure(proc) {
|
452 | var last_path = '';
|
453 | var current_path = proc.pm2_env.cwd || path.dirname(proc.pm2_env.pm_exec_path);
|
454 | var proc_id = proc.pm2_env.pm_id;
|
455 |
|
456 | proc.pm2_env.version = Utility.findPackageVersion(proc.pm2_env.pm_exec_path || proc.pm2_env.cwd);
|
457 |
|
458 | if (proc.pm2_env.vizion_running === true) {
|
459 | debug('Vizion is already running for proc id: %d, skipping this round', proc_id);
|
460 | return God.notify('online', proc);
|
461 | }
|
462 | proc.pm2_env.vizion_running = true;
|
463 |
|
464 | vizion.analyze({folder : current_path}, function recur_path(err, meta){
|
465 | var proc = God.clusters_db[proc_id];
|
466 |
|
467 | if (err)
|
468 | debug(err.stack || err);
|
469 |
|
470 | if (!proc ||
|
471 | !proc.pm2_env ||
|
472 | proc.pm2_env.status == cst.STOPPED_STATUS ||
|
473 | proc.pm2_env.status == cst.STOPPING_STATUS) {
|
474 | return console.error('Cancelling versioning data parsing');
|
475 | }
|
476 |
|
477 | proc.pm2_env.vizion_running = false;
|
478 |
|
479 | if (!err) {
|
480 | proc.pm2_env.versioning = meta;
|
481 | proc.pm2_env.versioning.repo_path = current_path;
|
482 | God.notify('online', proc);
|
483 | }
|
484 | else if (err && current_path === last_path) {
|
485 | proc.pm2_env.versioning = null;
|
486 | God.notify('online', proc);
|
487 | }
|
488 | else {
|
489 | last_path = current_path;
|
490 | current_path = path.dirname(current_path);
|
491 | proc.pm2_env.vizion_running = true;
|
492 | vizion.analyze({folder : current_path}, recur_path);
|
493 | }
|
494 | return false;
|
495 | });
|
496 | };
|
497 |
|
498 |
|
499 |
|
500 |
|
501 |
|
502 |
|
503 | God.injectVariables = function injectVariables (env, cb) {
|
504 |
|
505 | var instanceKey = process.env.PM2_PROCESS_INSTANCE_VAR || env.instance_var;
|
506 |
|
507 |
|
508 | var instances = Object.keys(God.clusters_db)
|
509 | .map(function (procId) {
|
510 | return God.clusters_db[procId];
|
511 | }).filter(function (proc) {
|
512 | return proc.pm2_env.name === env.name &&
|
513 | typeof proc.pm2_env[instanceKey] !== 'undefined';
|
514 | }).map(function (proc) {
|
515 | return proc.pm2_env[instanceKey];
|
516 | }).sort(function (a, b) {
|
517 | return b - a;
|
518 | });
|
519 |
|
520 | var instanceNumber = typeof instances[0] === 'undefined' ? 0 : instances[0] + 1;
|
521 |
|
522 | for (var i = 0; i < instances.length; i++) {
|
523 | if (instances.indexOf(i) === -1) {
|
524 | instanceNumber = i;
|
525 | break;
|
526 | }
|
527 | }
|
528 | env[instanceKey] = instanceNumber;
|
529 |
|
530 |
|
531 | if (env.increment_var) {
|
532 | var lastIncrement = Object.keys(God.clusters_db)
|
533 | .map(function (procId) {
|
534 | return God.clusters_db[procId];
|
535 | }).filter(function (proc) {
|
536 | return proc.pm2_env.name === env.name &&
|
537 | typeof proc.pm2_env[env.increment_var] !== 'undefined';
|
538 | }).map(function (proc) {
|
539 | return proc.pm2_env[env.increment_var];
|
540 | }).sort(function (a, b) {
|
541 | return b - a;
|
542 | })[0];
|
543 |
|
544 | var defaut = env.env[env.increment_var] || 0;
|
545 | env[env.increment_var] = typeof lastIncrement === 'undefined' ? defaut : lastIncrement + 1;
|
546 | env.env[env.increment_var] = env[env.increment_var];
|
547 | }
|
548 |
|
549 | return cb(null, env);
|
550 | };
|
551 |
|
552 | God.monitor = function Monitor(pm_id, cb) {
|
553 | if (!God.clusters_db[pm_id] || !God.clusters_db[pm_id].pm2_env)
|
554 | return cb(new Error('Unknown pm_id'));
|
555 |
|
556 | God.clusters_db[pm_id].pm2_env._km_monitored = true;
|
557 | return cb(null, { success : true, pm_id : pm_id });
|
558 | }
|
559 |
|
560 | God.unmonitor = function Monitor(pm_id, cb) {
|
561 | if (!God.clusters_db[pm_id] || !God.clusters_db[pm_id].pm2_env)
|
562 | return cb(new Error('Unknown pm_id'));
|
563 |
|
564 | God.clusters_db[pm_id].pm2_env._km_monitored = false;
|
565 | return cb(null, { success : true, pm_id : pm_id });
|
566 | }
|
567 |
|
568 |
|
569 |
|
570 |
|
571 | require('./Worker.js')(God);
|
572 | setTimeout(function() {
|
573 | God.Worker.start();
|
574 | }, 500);
|