1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 | var cluster = require('cluster');
|
18 | var numCPUs = require('os').cpus() ? require('os').cpus().length : 1;
|
19 | var path = require('path');
|
20 | var EventEmitter2 = require('eventemitter2').EventEmitter2;
|
21 | var fs = require('fs');
|
22 | var vizion = require('vizion');
|
23 | var debug = require('debug')('pm2:god');
|
24 | var Utility = require('./Utility');
|
25 | var cst = require('../constants.js');
|
26 | var timesLimit = require('async/timesLimit');
|
27 | var Configuration = require('./Configuration.js');
|
28 | var semver = require('semver');
|
29 |
|
30 |
|
31 |
|
32 |
|
33 | if (semver.lt(process.version, '10.0.0')) {
|
34 | cluster.setupMaster({
|
35 | windowsHide: true,
|
36 | exec : path.resolve(path.dirname(module.filename), 'ProcessContainerLegacy.js')
|
37 | });
|
38 | }
|
39 | else {
|
40 | cluster.setupMaster({
|
41 | windowsHide: true,
|
42 | exec : path.resolve(path.dirname(module.filename), 'ProcessContainer.js')
|
43 | });
|
44 | }
|
45 |
|
46 |
|
47 |
|
48 |
|
49 | var God = module.exports = {
|
50 | next_id : 0,
|
51 | clusters_db : {},
|
52 | configuration: {},
|
53 | started_at : Date.now(),
|
54 | system_infos_proc: null,
|
55 | system_infos: null,
|
56 | bus : new EventEmitter2({
|
57 | wildcard: true,
|
58 | delimiter: ':',
|
59 | maxListeners: 1000
|
60 | })
|
61 | };
|
62 |
|
63 | Utility.overrideConsole(God.bus);
|
64 |
|
65 |
|
66 |
|
67 |
|
68 | require('./Event.js')(God);
|
69 | require('./God/Methods.js')(God);
|
70 | require('./God/ForkMode.js')(God);
|
71 | require('./God/ClusterMode.js')(God);
|
72 | require('./God/Reload')(God);
|
73 | require('./God/ActionMethods')(God);
|
74 | require('./Watcher')(God);
|
75 |
|
76 | God.init = function() {
|
77 | require('./Worker.js')(this)
|
78 | God.system_infos_proc = null
|
79 |
|
80 | this.configuration = Configuration.getSync('pm2')
|
81 |
|
82 | if (this.configuration && this.configuration.sysmonit == 'true') {
|
83 | God.launchSysMonitoring({}, () => { console.log('System monitoring launched') })
|
84 | }
|
85 |
|
86 | setTimeout(function() {
|
87 | God.Worker.start()
|
88 | }, 500)
|
89 | }
|
90 |
|
91 | God.writeExitSeparator = function(pm2_env, code, signal) {
|
92 | try {
|
93 | var exit_sep = `[PM2][${new Date().toISOString()}] app exited`
|
94 | if (code)
|
95 | exit_sep += `itself with exit code: ${code}`
|
96 | if (signal)
|
97 | exit_sep += `by an external signal: ${signal}`
|
98 | exit_sep += '\n'
|
99 |
|
100 | if (pm2_env.pm_out_log_path)
|
101 | fs.writeFileSync(pm2_env.pm_out_log_path, exit_sep)
|
102 | if (pm2_env.pm_err_log_path)
|
103 | fs.writeFileSync(pm2_env.pm_err_log_path, exit_sep)
|
104 | if (pm2_env.pm_log_path)
|
105 | fs.writeFileSync(pm2_env.pm_log_path, exit_sep)
|
106 | } catch(e) {
|
107 | }
|
108 | }
|
109 |
|
110 |
|
111 |
|
112 |
|
113 | God.prepare = function prepare (env, cb) {
|
114 |
|
115 | env.env.unique_id = Utility.generateUUID()
|
116 |
|
117 |
|
118 | if (typeof env.instances === 'undefined') {
|
119 | env.vizion_running = false;
|
120 | if (env.env && env.env.vizion_running) env.env.vizion_running = false;
|
121 |
|
122 | if (env.status == cst.STOPPED_STATUS) {
|
123 | env.pm_id = God.getNewId()
|
124 | var clu = {
|
125 | pm2_env : env,
|
126 | process: {
|
127 | }
|
128 | }
|
129 | God.clusters_db[env.pm_id] = clu
|
130 | return cb(null, [ God.clusters_db[env.pm_id] ])
|
131 | }
|
132 |
|
133 | return God.executeApp(env, function (err, clu) {
|
134 | if (err) return cb(err);
|
135 | God.notify('start', clu, true);
|
136 | return cb(null, [ Utility.clone(clu) ]);
|
137 | });
|
138 | }
|
139 |
|
140 |
|
141 | env.instances = parseInt(env.instances);
|
142 | if (env.instances === 0) {
|
143 | env.instances = numCPUs;
|
144 | } else if (env.instances < 0) {
|
145 | env.instances += numCPUs;
|
146 | }
|
147 | if (env.instances <= 0) {
|
148 | env.instances = 1;
|
149 | }
|
150 |
|
151 | timesLimit(env.instances, 1, function (n, next) {
|
152 | env.vizion_running = false;
|
153 | if (env.env && env.env.vizion_running) {
|
154 | env.env.vizion_running = false;
|
155 | }
|
156 |
|
157 | God.injectVariables(env, function inject (err, _env) {
|
158 | if (err) return next(err);
|
159 | return God.executeApp(Utility.clone(_env), function (err, clu) {
|
160 | if (err) return next(err);
|
161 | God.notify('start', clu, true);
|
162 |
|
163 |
|
164 | return next(null, Utility.clone(clu));
|
165 | });
|
166 | });
|
167 | }, cb);
|
168 | };
|
169 |
|
170 |
|
171 |
|
172 |
|
173 |
|
174 |
|
175 |
|
176 |
|
177 |
|
178 | God.executeApp = function executeApp(env, cb) {
|
179 | var env_copy = Utility.clone(env);
|
180 |
|
181 | Utility.extend(env_copy, env_copy.env);
|
182 |
|
183 | env_copy['status'] = cst.LAUNCHING_STATUS;
|
184 | env_copy['pm_uptime'] = Date.now();
|
185 | env_copy['axm_actions'] = [];
|
186 | env_copy['axm_monitor'] = {};
|
187 | env_copy['axm_options'] = {};
|
188 | env_copy['axm_dynamic'] = {};
|
189 | env_copy['vizion_running'] =
|
190 | env_copy['vizion_running'] !== undefined ? env_copy['vizion_running'] : false;
|
191 |
|
192 | if (!env_copy.created_at)
|
193 | env_copy['created_at'] = Date.now();
|
194 |
|
195 | |
196 |
|
197 |
|
198 |
|
199 |
|
200 |
|
201 |
|
202 | if (env_copy['pm_id'] === undefined) {
|
203 | env_copy['pm_id'] = God.getNewId();
|
204 | env_copy['restart_time'] = 0;
|
205 | env_copy['unstable_restarts'] = 0;
|
206 |
|
207 |
|
208 | env_copy.pm_pid_path = env_copy.pm_pid_path.replace(/-[0-9]+\.pid$|\.pid$/g, '-' + env_copy['pm_id'] + '.pid');
|
209 |
|
210 |
|
211 | if (!env_copy['merge_logs']) {
|
212 | ['', '_out', '_err'].forEach(function(k){
|
213 | var key = 'pm' + k + '_log_path';
|
214 | env_copy[key] && (env_copy[key] = env_copy[key].replace(/-[0-9]+\.log$|\.log$/g, '-' + env_copy['pm_id'] + '.log'));
|
215 | });
|
216 | }
|
217 |
|
218 |
|
219 | if (env_copy['watch']) {
|
220 | God.watch.enable(env_copy);
|
221 | }
|
222 | }
|
223 |
|
224 | God.registerCron(env_copy)
|
225 |
|
226 |
|
227 | var readyCb = function ready(proc) {
|
228 |
|
229 | if (proc.pm2_env.vizion !== false && proc.pm2_env.vizion !== "false")
|
230 | God.finalizeProcedure(proc);
|
231 | else
|
232 | God.notify('online', proc);
|
233 |
|
234 | if (proc.pm2_env.status !== cst.ERRORED_STATUS)
|
235 | proc.pm2_env.status = cst.ONLINE_STATUS
|
236 |
|
237 | console.log(`App [${proc.pm2_env.name}:${proc.pm2_env.pm_id}] online`);
|
238 | if (cb) cb(null, proc);
|
239 | }
|
240 |
|
241 | if (env_copy.exec_mode === 'cluster_mode') {
|
242 | |
243 |
|
244 |
|
245 | God.nodeApp(env_copy, function nodeApp(err, clu) {
|
246 | if (cb && err) return cb(err);
|
247 | if (err) return false;
|
248 |
|
249 | var old_env = God.clusters_db[clu.pm2_env.pm_id];
|
250 |
|
251 | if (old_env) {
|
252 | old_env = null;
|
253 | God.clusters_db[clu.pm2_env.pm_id] = null;
|
254 | }
|
255 |
|
256 | God.clusters_db[clu.pm2_env.pm_id] = clu;
|
257 |
|
258 | clu.once('error', function(err) {
|
259 | console.error(err.stack || err);
|
260 | clu.pm2_env.status = cst.ERRORED_STATUS;
|
261 | try {
|
262 | clu.destroy && clu.destroy();
|
263 | }
|
264 | catch (e) {
|
265 | console.error(e.stack || e);
|
266 | God.handleExit(clu, cst.ERROR_EXIT);
|
267 | }
|
268 | });
|
269 |
|
270 | clu.once('disconnect', function() {
|
271 | console.log('App name:%s id:%s disconnected', clu.pm2_env.name, clu.pm2_env.pm_id);
|
272 | });
|
273 |
|
274 | clu.once('exit', function cluExit(code, signal) {
|
275 |
|
276 | God.handleExit(clu, code || 0, signal || 'SIGINT');
|
277 | });
|
278 |
|
279 | return clu.once('online', function () {
|
280 | if (!clu.pm2_env.wait_ready)
|
281 | return readyCb(clu);
|
282 |
|
283 |
|
284 | var ready_timeout = setTimeout(function() {
|
285 | God.bus.removeListener('process:msg', listener)
|
286 | return readyCb(clu)
|
287 | }, clu.pm2_env.listen_timeout || cst.GRACEFUL_LISTEN_TIMEOUT);
|
288 |
|
289 | var listener = function (packet) {
|
290 | if (packet.raw === 'ready' &&
|
291 | packet.process.name === clu.pm2_env.name &&
|
292 | packet.process.pm_id === clu.pm2_env.pm_id) {
|
293 | clearTimeout(ready_timeout);
|
294 | God.bus.removeListener('process:msg', listener)
|
295 | return readyCb(clu)
|
296 | }
|
297 | }
|
298 |
|
299 | God.bus.on('process:msg', listener);
|
300 | });
|
301 | });
|
302 | }
|
303 | else {
|
304 | |
305 |
|
306 |
|
307 | God.forkMode(env_copy, function forkMode(err, clu) {
|
308 | if (cb && err) return cb(err);
|
309 | if (err) return false;
|
310 |
|
311 | var old_env = God.clusters_db[clu.pm2_env.pm_id];
|
312 | if (old_env) old_env = null;
|
313 |
|
314 | God.clusters_db[env_copy.pm_id] = clu;
|
315 |
|
316 | clu.once('error', function cluError(err) {
|
317 | console.error(err.stack || err);
|
318 | clu.pm2_env.status = cst.ERRORED_STATUS;
|
319 | try {
|
320 | clu.kill && clu.kill();
|
321 | }
|
322 | catch (e) {
|
323 | console.error(e.stack || e);
|
324 | God.handleExit(clu, cst.ERROR_EXIT);
|
325 | }
|
326 | });
|
327 |
|
328 | clu.once('exit', function cluClose(code, signal) {
|
329 |
|
330 |
|
331 | if (clu.connected === true)
|
332 | clu.disconnect && clu.disconnect();
|
333 | clu._reloadLogs = null;
|
334 | return God.handleExit(clu, code || 0, signal);
|
335 | });
|
336 |
|
337 | if (!clu.pm2_env.wait_ready)
|
338 | return readyCb(clu);
|
339 |
|
340 |
|
341 | var ready_timeout = setTimeout(function() {
|
342 | God.bus.removeListener('process:msg', listener)
|
343 | return readyCb(clu)
|
344 | }, clu.pm2_env.listen_timeout || cst.GRACEFUL_LISTEN_TIMEOUT);
|
345 |
|
346 | var listener = function (packet) {
|
347 | if (packet.raw === 'ready' &&
|
348 | packet.process.name === clu.pm2_env.name &&
|
349 | packet.process.pm_id === clu.pm2_env.pm_id) {
|
350 | clearTimeout(ready_timeout);
|
351 | God.bus.removeListener('process:msg', listener)
|
352 | return readyCb(clu)
|
353 | }
|
354 | }
|
355 | God.bus.on('process:msg', listener);
|
356 | });
|
357 | }
|
358 | return false;
|
359 | };
|
360 |
|
361 |
|
362 |
|
363 |
|
364 |
|
365 |
|
366 |
|
367 |
|
368 | God.handleExit = function handleExit(clu, exit_code, kill_signal) {
|
369 | console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] exited with code [${exit_code}] via signal [${kill_signal || 'SIGINT'}]`)
|
370 |
|
371 | var proc = this.clusters_db[clu.pm2_env.pm_id];
|
372 |
|
373 | if (!proc) {
|
374 | console.error('Process undefined ? with process id ', clu.pm2_env.pm_id);
|
375 | return false;
|
376 | }
|
377 |
|
378 | var stopping = (proc.pm2_env.status == cst.STOPPING_STATUS
|
379 | || proc.pm2_env.status == cst.STOPPED_STATUS
|
380 | || proc.pm2_env.status == cst.ERRORED_STATUS)
|
381 | || (proc.pm2_env.autorestart === false || proc.pm2_env.autorestart === "false");
|
382 |
|
383 | var overlimit = false;
|
384 |
|
385 | if (stopping) proc.process.pid = 0;
|
386 |
|
387 |
|
388 | if (proc.pm2_env.axm_actions) proc.pm2_env.axm_actions = [];
|
389 | if (proc.pm2_env.axm_monitor) proc.pm2_env.axm_monitor = {};
|
390 |
|
391 | if (proc.pm2_env.status != cst.ERRORED_STATUS &&
|
392 | proc.pm2_env.status != cst.STOPPING_STATUS)
|
393 | proc.pm2_env.status = cst.STOPPED_STATUS;
|
394 |
|
395 | if (proc.pm2_env.pm_id.toString().indexOf('_old_') !== 0) {
|
396 | try {
|
397 | fs.unlinkSync(proc.pm2_env.pm_pid_path);
|
398 | } catch (e) {
|
399 | debug('Error when unlinking pid file', e);
|
400 | }
|
401 | }
|
402 |
|
403 | |
404 |
|
405 |
|
406 |
|
407 |
|
408 |
|
409 | var min_uptime = typeof(proc.pm2_env.min_uptime) !== 'undefined' ? proc.pm2_env.min_uptime : 1000;
|
410 | var max_restarts = typeof(proc.pm2_env.max_restarts) !== 'undefined' ? proc.pm2_env.max_restarts : 16;
|
411 |
|
412 | if ((Date.now() - proc.pm2_env.created_at) < (min_uptime * max_restarts)) {
|
413 | if ((Date.now() - proc.pm2_env.pm_uptime) < min_uptime) {
|
414 |
|
415 | proc.pm2_env.unstable_restarts += 1;
|
416 | }
|
417 | }
|
418 |
|
419 |
|
420 | if (proc.pm2_env.unstable_restarts >= max_restarts) {
|
421 |
|
422 |
|
423 |
|
424 | proc.pm2_env.status = cst.ERRORED_STATUS;
|
425 | proc.process.pid = 0;
|
426 |
|
427 | console.log('Script %s had too many unstable restarts (%d). Stopped. %j',
|
428 | proc.pm2_env.pm_exec_path,
|
429 | proc.pm2_env.unstable_restarts,
|
430 | proc.pm2_env.status);
|
431 |
|
432 | God.notify('restart overlimit', proc);
|
433 |
|
434 | proc.pm2_env.unstable_restarts = 0;
|
435 | proc.pm2_env.created_at = null;
|
436 | overlimit = true;
|
437 | }
|
438 |
|
439 | if (typeof(exit_code) !== 'undefined') proc.pm2_env.exit_code = exit_code;
|
440 |
|
441 | God.notify('exit', proc);
|
442 |
|
443 | if (God.pm2_being_killed) {
|
444 |
|
445 | return false;
|
446 | }
|
447 |
|
448 | var restart_delay = 0;
|
449 |
|
450 | if (proc.pm2_env.restart_delay !== undefined &&
|
451 | !isNaN(parseInt(proc.pm2_env.restart_delay))) {
|
452 | proc.pm2_env.status = cst.WAITING_RESTART;
|
453 | restart_delay = parseInt(proc.pm2_env.restart_delay);
|
454 | }
|
455 |
|
456 | if (proc.pm2_env.exp_backoff_restart_delay !== undefined &&
|
457 | !isNaN(parseInt(proc.pm2_env.exp_backoff_restart_delay))) {
|
458 | proc.pm2_env.status = cst.WAITING_RESTART;
|
459 | if (!proc.pm2_env.prev_restart_delay) {
|
460 | proc.pm2_env.prev_restart_delay = proc.pm2_env.exp_backoff_restart_delay
|
461 | restart_delay = proc.pm2_env.exp_backoff_restart_delay
|
462 | }
|
463 | else {
|
464 | proc.pm2_env.prev_restart_delay = Math.floor(Math.min(15000, proc.pm2_env.prev_restart_delay * 1.5))
|
465 | restart_delay = proc.pm2_env.prev_restart_delay
|
466 | }
|
467 | console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] will restart in ${restart_delay}ms`)
|
468 | }
|
469 |
|
470 | if (!stopping && !overlimit) {
|
471 |
|
472 | Object.defineProperty(proc.pm2_env, 'restart_task', {configurable: true, writable: true});
|
473 | proc.pm2_env.restart_task = setTimeout(function() {
|
474 | proc.pm2_env.restart_time += 1;
|
475 | God.executeApp(proc.pm2_env);
|
476 | }, restart_delay);
|
477 | }
|
478 |
|
479 | return false;
|
480 | };
|
481 |
|
482 |
|
483 |
|
484 |
|
485 |
|
486 |
|
487 | God.finalizeProcedure = function finalizeProcedure(proc) {
|
488 | var last_path = '';
|
489 | var current_path = proc.pm2_env.cwd || path.dirname(proc.pm2_env.pm_exec_path);
|
490 | var proc_id = proc.pm2_env.pm_id;
|
491 |
|
492 | proc.pm2_env.version = Utility.findPackageVersion(proc.pm2_env.pm_exec_path || proc.pm2_env.cwd);
|
493 |
|
494 | if (proc.pm2_env.vizion_running === true) {
|
495 | debug('Vizion is already running for proc id: %d, skipping this round', proc_id);
|
496 | return God.notify('online', proc);
|
497 | }
|
498 | proc.pm2_env.vizion_running = true;
|
499 |
|
500 | vizion.analyze({folder : current_path}, function recur_path(err, meta){
|
501 | var proc = God.clusters_db[proc_id];
|
502 |
|
503 | if (err)
|
504 | debug(err.stack || err);
|
505 |
|
506 | if (!proc ||
|
507 | !proc.pm2_env ||
|
508 | proc.pm2_env.status == cst.STOPPED_STATUS ||
|
509 | proc.pm2_env.status == cst.STOPPING_STATUS ||
|
510 | proc.pm2_env.status == cst.ERRORED_STATUS) {
|
511 | return console.error('Cancelling versioning data parsing');
|
512 | }
|
513 |
|
514 | proc.pm2_env.vizion_running = false;
|
515 |
|
516 | if (!err) {
|
517 | proc.pm2_env.versioning = meta;
|
518 | proc.pm2_env.versioning.repo_path = current_path;
|
519 | God.notify('online', proc);
|
520 | }
|
521 | else if (err && current_path === last_path) {
|
522 | proc.pm2_env.versioning = null;
|
523 | God.notify('online', proc);
|
524 | }
|
525 | else {
|
526 | last_path = current_path;
|
527 | current_path = path.dirname(current_path);
|
528 | proc.pm2_env.vizion_running = true;
|
529 | vizion.analyze({folder : current_path}, recur_path);
|
530 | }
|
531 | return false;
|
532 | });
|
533 | };
|
534 |
|
535 |
|
536 |
|
537 |
|
538 |
|
539 |
|
540 | God.injectVariables = function injectVariables (env, cb) {
|
541 |
|
542 | var instanceKey = process.env.PM2_PROCESS_INSTANCE_VAR || env.instance_var;
|
543 |
|
544 |
|
545 | var instances = Object.keys(God.clusters_db)
|
546 | .map(function (procId) {
|
547 | return God.clusters_db[procId];
|
548 | }).filter(function (proc) {
|
549 | return proc.pm2_env.name === env.name &&
|
550 | typeof proc.pm2_env[instanceKey] !== 'undefined';
|
551 | }).map(function (proc) {
|
552 | return proc.pm2_env[instanceKey];
|
553 | }).sort(function (a, b) {
|
554 | return b - a;
|
555 | });
|
556 |
|
557 | var instanceNumber = typeof instances[0] === 'undefined' ? 0 : instances[0] + 1;
|
558 |
|
559 | for (var i = 0; i < instances.length; i++) {
|
560 | if (instances.indexOf(i) === -1) {
|
561 | instanceNumber = i;
|
562 | break;
|
563 | }
|
564 | }
|
565 | env[instanceKey] = instanceNumber;
|
566 |
|
567 |
|
568 | if (env.increment_var) {
|
569 | var lastIncrement = Object.keys(God.clusters_db)
|
570 | .map(function (procId) {
|
571 | return God.clusters_db[procId];
|
572 | }).filter(function (proc) {
|
573 | return proc.pm2_env.name === env.name &&
|
574 | typeof proc.pm2_env[env.increment_var] !== 'undefined';
|
575 | }).map(function (proc) {
|
576 | return proc.pm2_env[env.increment_var];
|
577 | }).sort(function (a, b) {
|
578 | return b - a;
|
579 | })[0];
|
580 |
|
581 | var defaut = env.env[env.increment_var] || 0;
|
582 | env[env.increment_var] = typeof lastIncrement === 'undefined' ? defaut : lastIncrement + 1;
|
583 | env.env[env.increment_var] = env[env.increment_var];
|
584 | }
|
585 |
|
586 | return cb(null, env);
|
587 | };
|
588 |
|
589 | God.launchSysMonitoring = function(env, cb) {
|
590 | if (God.system_infos_proc !== null)
|
591 | return cb(new Error('Sys Monitoring already launched'))
|
592 |
|
593 | try {
|
594 | var sysinfo = require('./Sysinfo/SystemInfo.js')
|
595 | God.system_infos_proc = new sysinfo()
|
596 |
|
597 | setInterval(() => {
|
598 | God.system_infos_proc.query((err, data) => {
|
599 | if (err) return
|
600 | God.system_infos = data
|
601 | })
|
602 | }, 1000)
|
603 |
|
604 | God.system_infos_proc.fork()
|
605 | } catch(e) {
|
606 | console.log(e)
|
607 | God.system_infos_proc = null
|
608 | }
|
609 | return cb()
|
610 | }
|
611 |
|
612 | God.init()
|