1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 | var cluster = require('cluster');
|
18 | var numCPUs = require('os').cpus() ? require('os').cpus().length : 1;
|
19 | var path = require('path');
|
20 | var EventEmitter2 = require('eventemitter2').EventEmitter2;
|
21 | var fs = require('fs');
|
22 | var vizion = require('vizion');
|
23 | var debug = require('debug')('pm2:god');
|
24 | var Utility = require('./Utility');
|
25 | var cst = require('../constants.js');
|
26 | var timesLimit = require('async/timesLimit');
|
27 | var Configuration = require('./Configuration.js');
|
28 | var semver = require('semver');
|
29 |
|
30 |
|
31 |
|
32 |
|
33 | if (semver.lt(process.version, '10.0.0')) {
|
34 | cluster.setupMaster({
|
35 | windowsHide: true,
|
36 | exec : path.resolve(path.dirname(module.filename), 'ProcessContainerLegacy.js')
|
37 | });
|
38 | }
|
39 | else {
|
40 | cluster.setupMaster({
|
41 | windowsHide: true,
|
42 | exec : path.resolve(path.dirname(module.filename), 'ProcessContainer.js')
|
43 | });
|
44 | }
|
45 |
|
46 |
|
47 |
|
48 |
|
49 | var God = module.exports = {
|
50 | next_id : 0,
|
51 | clusters_db : {},
|
52 | configuration: {},
|
53 | started_at : Date.now(),
|
54 | system_infos_proc: null,
|
55 | system_infos: null,
|
56 | bus : new EventEmitter2({
|
57 | wildcard: true,
|
58 | delimiter: ':',
|
59 | maxListeners: 1000
|
60 | })
|
61 | };
|
62 |
|
63 | Utility.overrideConsole(God.bus);
|
64 |
|
65 |
|
66 |
|
67 |
|
68 | require('./Event.js')(God);
|
69 | require('./God/Methods.js')(God);
|
70 | require('./God/ForkMode.js')(God);
|
71 | require('./God/ClusterMode.js')(God);
|
72 | require('./God/Reload')(God);
|
73 | require('./God/ActionMethods')(God);
|
74 | require('./Watcher')(God);
|
75 |
|
76 | God.init = function() {
|
77 | require('./Worker.js')(this)
|
78 | God.system_infos_proc = null
|
79 |
|
80 | this.configuration = Configuration.getSync('pm2')
|
81 |
|
82 | setTimeout(function() {
|
83 | God.Worker.start()
|
84 | }, 500)
|
85 | }
|
86 |
|
87 | God.writeExitSeparator = function(pm2_env, code, signal) {
|
88 | try {
|
89 | var exit_sep = `[PM2][${new Date().toISOString()}] app exited`
|
90 | if (code)
|
91 | exit_sep += `itself with exit code: ${code}`
|
92 | if (signal)
|
93 | exit_sep += `by an external signal: ${signal}`
|
94 | exit_sep += '\n'
|
95 |
|
96 | if (pm2_env.pm_out_log_path)
|
97 | fs.writeFileSync(pm2_env.pm_out_log_path, exit_sep)
|
98 | if (pm2_env.pm_err_log_path)
|
99 | fs.writeFileSync(pm2_env.pm_err_log_path, exit_sep)
|
100 | if (pm2_env.pm_log_path)
|
101 | fs.writeFileSync(pm2_env.pm_log_path, exit_sep)
|
102 | } catch(e) {
|
103 | }
|
104 | }
|
105 |
|
106 |
|
107 |
|
108 |
|
109 | God.prepare = function prepare (env, cb) {
|
110 |
|
111 | env.env.unique_id = Utility.generateUUID()
|
112 |
|
113 |
|
114 | if (typeof env.instances === 'undefined') {
|
115 | env.vizion_running = false;
|
116 | if (env.env && env.env.vizion_running) env.env.vizion_running = false;
|
117 |
|
118 | if (env.status == cst.STOPPED_STATUS) {
|
119 | env.pm_id = God.getNewId()
|
120 | var clu = {
|
121 | pm2_env : env,
|
122 | process: {
|
123 | }
|
124 | }
|
125 | God.clusters_db[env.pm_id] = clu
|
126 | return cb(null, [ God.clusters_db[env.pm_id] ])
|
127 | }
|
128 |
|
129 | return God.executeApp(env, function (err, clu) {
|
130 | if (err) return cb(err);
|
131 | God.notify('start', clu, true);
|
132 | return cb(null, [ Utility.clone(clu) ]);
|
133 | });
|
134 | }
|
135 |
|
136 |
|
137 | env.instances = parseInt(env.instances);
|
138 | if (env.instances === 0) {
|
139 | env.instances = numCPUs;
|
140 | } else if (env.instances < 0) {
|
141 | env.instances += numCPUs;
|
142 | }
|
143 | if (env.instances <= 0) {
|
144 | env.instances = 1;
|
145 | }
|
146 |
|
147 | timesLimit(env.instances, 1, function (n, next) {
|
148 | env.vizion_running = false;
|
149 | if (env.env && env.env.vizion_running) {
|
150 | env.env.vizion_running = false;
|
151 | }
|
152 |
|
153 | God.injectVariables(env, function inject (err, _env) {
|
154 | if (err) return next(err);
|
155 | return God.executeApp(Utility.clone(_env), function (err, clu) {
|
156 | if (err) return next(err);
|
157 | God.notify('start', clu, true);
|
158 |
|
159 |
|
160 | return next(null, Utility.clone(clu));
|
161 | });
|
162 | });
|
163 | }, cb);
|
164 | };
|
165 |
|
166 |
|
167 |
|
168 |
|
169 |
|
170 |
|
171 |
|
172 |
|
173 |
|
174 | God.executeApp = function executeApp(env, cb) {
|
175 | var env_copy = Utility.clone(env);
|
176 |
|
177 | Utility.extend(env_copy, env_copy.env);
|
178 |
|
179 | env_copy['status'] = cst.LAUNCHING_STATUS;
|
180 | env_copy['pm_uptime'] = Date.now();
|
181 | env_copy['axm_actions'] = [];
|
182 | env_copy['axm_monitor'] = {};
|
183 | env_copy['axm_options'] = {};
|
184 | env_copy['axm_dynamic'] = {};
|
185 | env_copy['vizion_running'] =
|
186 | env_copy['vizion_running'] !== undefined ? env_copy['vizion_running'] : false;
|
187 |
|
188 | if (!env_copy.created_at)
|
189 | env_copy['created_at'] = Date.now();
|
190 |
|
191 | |
192 |
|
193 |
|
194 |
|
195 |
|
196 |
|
197 |
|
198 | if (env_copy['pm_id'] === undefined) {
|
199 | env_copy['pm_id'] = God.getNewId();
|
200 | env_copy['restart_time'] = 0;
|
201 | env_copy['unstable_restarts'] = 0;
|
202 |
|
203 |
|
204 | env_copy.pm_pid_path = env_copy.pm_pid_path.replace(/-[0-9]+\.pid$|\.pid$/g, '-' + env_copy['pm_id'] + '.pid');
|
205 |
|
206 |
|
207 | if (!env_copy['merge_logs']) {
|
208 | ['', '_out', '_err'].forEach(function(k){
|
209 | var key = 'pm' + k + '_log_path';
|
210 | env_copy[key] && (env_copy[key] = env_copy[key].replace(/-[0-9]+\.log$|\.log$/g, '-' + env_copy['pm_id'] + '.log'));
|
211 | });
|
212 | }
|
213 |
|
214 |
|
215 | if (env_copy['watch']) {
|
216 | God.watch.enable(env_copy);
|
217 | }
|
218 | }
|
219 |
|
220 | God.registerCron(env_copy)
|
221 |
|
222 |
|
223 | var readyCb = function ready(proc) {
|
224 |
|
225 | if (proc.pm2_env.vizion !== false && proc.pm2_env.vizion !== "false")
|
226 | God.finalizeProcedure(proc);
|
227 | else
|
228 | God.notify('online', proc);
|
229 |
|
230 | if (proc.pm2_env.status !== cst.ERRORED_STATUS)
|
231 | proc.pm2_env.status = cst.ONLINE_STATUS
|
232 |
|
233 | console.log(`App [${proc.pm2_env.name}:${proc.pm2_env.pm_id}] online`);
|
234 | if (cb) cb(null, proc);
|
235 | }
|
236 |
|
237 | if (env_copy.exec_mode === 'cluster_mode') {
|
238 | |
239 |
|
240 |
|
241 | God.nodeApp(env_copy, function nodeApp(err, clu) {
|
242 | if (cb && err) return cb(err);
|
243 | if (err) return false;
|
244 |
|
245 | var old_env = God.clusters_db[clu.pm2_env.pm_id];
|
246 |
|
247 | if (old_env) {
|
248 | old_env = null;
|
249 | God.clusters_db[clu.pm2_env.pm_id] = null;
|
250 | }
|
251 |
|
252 | God.clusters_db[clu.pm2_env.pm_id] = clu;
|
253 |
|
254 | clu.once('error', function(err) {
|
255 | console.error(err.stack || err);
|
256 | clu.pm2_env.status = cst.ERRORED_STATUS;
|
257 | try {
|
258 | clu.destroy && clu.destroy();
|
259 | }
|
260 | catch (e) {
|
261 | console.error(e.stack || e);
|
262 | God.handleExit(clu, cst.ERROR_EXIT);
|
263 | }
|
264 | });
|
265 |
|
266 | clu.once('disconnect', function() {
|
267 | console.log('App name:%s id:%s disconnected', clu.pm2_env.name, clu.pm2_env.pm_id);
|
268 | });
|
269 |
|
270 | clu.once('exit', function cluExit(code, signal) {
|
271 |
|
272 | God.handleExit(clu, code || 0, signal || 'SIGINT');
|
273 | });
|
274 |
|
275 | return clu.once('online', function () {
|
276 | if (!clu.pm2_env.wait_ready)
|
277 | return readyCb(clu);
|
278 |
|
279 |
|
280 | var ready_timeout = setTimeout(function() {
|
281 | God.bus.removeListener('process:msg', listener)
|
282 | return readyCb(clu)
|
283 | }, clu.pm2_env.listen_timeout || cst.GRACEFUL_LISTEN_TIMEOUT);
|
284 |
|
285 | var listener = function (packet) {
|
286 | if (packet.raw === 'ready' &&
|
287 | packet.process.name === clu.pm2_env.name &&
|
288 | packet.process.pm_id === clu.pm2_env.pm_id) {
|
289 | clearTimeout(ready_timeout);
|
290 | God.bus.removeListener('process:msg', listener)
|
291 | return readyCb(clu)
|
292 | }
|
293 | }
|
294 |
|
295 | God.bus.on('process:msg', listener);
|
296 | });
|
297 | });
|
298 | }
|
299 | else {
|
300 | |
301 |
|
302 |
|
303 | God.forkMode(env_copy, function forkMode(err, clu) {
|
304 | if (cb && err) return cb(err);
|
305 | if (err) return false;
|
306 |
|
307 | var old_env = God.clusters_db[clu.pm2_env.pm_id];
|
308 | if (old_env) old_env = null;
|
309 |
|
310 | God.clusters_db[env_copy.pm_id] = clu;
|
311 |
|
312 | clu.once('error', function cluError(err) {
|
313 | console.error(err.stack || err);
|
314 | clu.pm2_env.status = cst.ERRORED_STATUS;
|
315 | try {
|
316 | clu.kill && clu.kill();
|
317 | }
|
318 | catch (e) {
|
319 | console.error(e.stack || e);
|
320 | God.handleExit(clu, cst.ERROR_EXIT);
|
321 | }
|
322 | });
|
323 |
|
324 | clu.once('exit', function cluClose(code, signal) {
|
325 |
|
326 |
|
327 | if (clu.connected === true)
|
328 | clu.disconnect && clu.disconnect();
|
329 | clu._reloadLogs = null;
|
330 | return God.handleExit(clu, code || 0, signal);
|
331 | });
|
332 |
|
333 | if (!clu.pm2_env.wait_ready)
|
334 | return readyCb(clu);
|
335 |
|
336 |
|
337 | var ready_timeout = setTimeout(function() {
|
338 | God.bus.removeListener('process:msg', listener)
|
339 | return readyCb(clu)
|
340 | }, clu.pm2_env.listen_timeout || cst.GRACEFUL_LISTEN_TIMEOUT);
|
341 |
|
342 | var listener = function (packet) {
|
343 | if (packet.raw === 'ready' &&
|
344 | packet.process.name === clu.pm2_env.name &&
|
345 | packet.process.pm_id === clu.pm2_env.pm_id) {
|
346 | clearTimeout(ready_timeout);
|
347 | God.bus.removeListener('process:msg', listener)
|
348 | return readyCb(clu)
|
349 | }
|
350 | }
|
351 | God.bus.on('process:msg', listener);
|
352 | });
|
353 | }
|
354 | return false;
|
355 | };
|
356 |
|
357 |
|
358 |
|
359 |
|
360 |
|
361 |
|
362 |
|
363 |
|
364 | God.handleExit = function handleExit(clu, exit_code, kill_signal) {
|
365 | console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] exited with code [${exit_code}] via signal [${kill_signal || 'SIGINT'}]`)
|
366 |
|
367 | var proc = this.clusters_db[clu.pm2_env.pm_id];
|
368 |
|
369 | if (!proc) {
|
370 | console.error('Process undefined ? with process id ', clu.pm2_env.pm_id);
|
371 | return false;
|
372 | }
|
373 |
|
374 | var stopping = (proc.pm2_env.status == cst.STOPPING_STATUS
|
375 | || proc.pm2_env.status == cst.STOPPED_STATUS
|
376 | || proc.pm2_env.status == cst.ERRORED_STATUS)
|
377 | || (proc.pm2_env.autorestart === false || proc.pm2_env.autorestart === "false");
|
378 |
|
379 | var overlimit = false;
|
380 |
|
381 | if (stopping) proc.process.pid = 0;
|
382 |
|
383 |
|
384 | if (proc.pm2_env.axm_actions) proc.pm2_env.axm_actions = [];
|
385 | if (proc.pm2_env.axm_monitor) proc.pm2_env.axm_monitor = {};
|
386 |
|
387 | if (proc.pm2_env.status != cst.ERRORED_STATUS &&
|
388 | proc.pm2_env.status != cst.STOPPING_STATUS)
|
389 | proc.pm2_env.status = cst.STOPPED_STATUS;
|
390 |
|
391 | if (proc.pm2_env.pm_id.toString().indexOf('_old_') !== 0) {
|
392 | try {
|
393 | fs.unlinkSync(proc.pm2_env.pm_pid_path);
|
394 | } catch (e) {
|
395 | debug('Error when unlinking pid file', e);
|
396 | }
|
397 | }
|
398 |
|
399 | |
400 |
|
401 |
|
402 |
|
403 |
|
404 |
|
405 | var min_uptime = typeof(proc.pm2_env.min_uptime) !== 'undefined' ? proc.pm2_env.min_uptime : 1000;
|
406 | var max_restarts = typeof(proc.pm2_env.max_restarts) !== 'undefined' ? proc.pm2_env.max_restarts : 16;
|
407 |
|
408 | if ((Date.now() - proc.pm2_env.created_at) < (min_uptime * max_restarts)) {
|
409 | if ((Date.now() - proc.pm2_env.pm_uptime) < min_uptime) {
|
410 |
|
411 | proc.pm2_env.unstable_restarts += 1;
|
412 | }
|
413 | }
|
414 |
|
415 |
|
416 | if (proc.pm2_env.unstable_restarts >= max_restarts) {
|
417 |
|
418 |
|
419 |
|
420 | proc.pm2_env.status = cst.ERRORED_STATUS;
|
421 | proc.process.pid = 0;
|
422 |
|
423 | console.log('Script %s had too many unstable restarts (%d). Stopped. %j',
|
424 | proc.pm2_env.pm_exec_path,
|
425 | proc.pm2_env.unstable_restarts,
|
426 | proc.pm2_env.status);
|
427 |
|
428 | God.notify('restart overlimit', proc);
|
429 |
|
430 | proc.pm2_env.unstable_restarts = 0;
|
431 | proc.pm2_env.created_at = null;
|
432 | overlimit = true;
|
433 | }
|
434 |
|
435 | if (typeof(exit_code) !== 'undefined') proc.pm2_env.exit_code = exit_code;
|
436 |
|
437 | God.notify('exit', proc);
|
438 |
|
439 | if (God.pm2_being_killed) {
|
440 |
|
441 | return false;
|
442 | }
|
443 |
|
444 | var restart_delay = 0;
|
445 |
|
446 | if (proc.pm2_env.restart_delay !== undefined &&
|
447 | !isNaN(parseInt(proc.pm2_env.restart_delay))) {
|
448 | proc.pm2_env.status = cst.WAITING_RESTART;
|
449 | restart_delay = parseInt(proc.pm2_env.restart_delay);
|
450 | }
|
451 |
|
452 | if (proc.pm2_env.exp_backoff_restart_delay !== undefined &&
|
453 | !isNaN(parseInt(proc.pm2_env.exp_backoff_restart_delay))) {
|
454 | proc.pm2_env.status = cst.WAITING_RESTART;
|
455 | if (!proc.pm2_env.prev_restart_delay) {
|
456 | proc.pm2_env.prev_restart_delay = proc.pm2_env.exp_backoff_restart_delay
|
457 | restart_delay = proc.pm2_env.exp_backoff_restart_delay
|
458 | }
|
459 | else {
|
460 | proc.pm2_env.prev_restart_delay = Math.floor(Math.min(15000, proc.pm2_env.prev_restart_delay * 1.5))
|
461 | restart_delay = proc.pm2_env.prev_restart_delay
|
462 | }
|
463 | console.log(`App [${clu.pm2_env.name}:${clu.pm2_env.pm_id}] will restart in ${restart_delay}ms`)
|
464 | }
|
465 |
|
466 | if (!stopping && !overlimit) {
|
467 |
|
468 | Object.defineProperty(proc.pm2_env, 'restart_task', {configurable: true, writable: true});
|
469 | proc.pm2_env.restart_task = setTimeout(function() {
|
470 | proc.pm2_env.restart_time += 1;
|
471 | God.executeApp(proc.pm2_env);
|
472 | }, restart_delay);
|
473 | }
|
474 |
|
475 | return false;
|
476 | };
|
477 |
|
478 |
|
479 |
|
480 |
|
481 |
|
482 |
|
483 | God.finalizeProcedure = function finalizeProcedure(proc) {
|
484 | var last_path = '';
|
485 | var current_path = proc.pm2_env.cwd || path.dirname(proc.pm2_env.pm_exec_path);
|
486 | var proc_id = proc.pm2_env.pm_id;
|
487 |
|
488 | proc.pm2_env.version = Utility.findPackageVersion(proc.pm2_env.pm_exec_path || proc.pm2_env.cwd);
|
489 |
|
490 | if (proc.pm2_env.vizion_running === true) {
|
491 | debug('Vizion is already running for proc id: %d, skipping this round', proc_id);
|
492 | return God.notify('online', proc);
|
493 | }
|
494 | proc.pm2_env.vizion_running = true;
|
495 |
|
496 | vizion.analyze({folder : current_path}, function recur_path(err, meta){
|
497 | var proc = God.clusters_db[proc_id];
|
498 |
|
499 | if (err)
|
500 | debug(err.stack || err);
|
501 |
|
502 | if (!proc ||
|
503 | !proc.pm2_env ||
|
504 | proc.pm2_env.status == cst.STOPPED_STATUS ||
|
505 | proc.pm2_env.status == cst.STOPPING_STATUS ||
|
506 | proc.pm2_env.status == cst.ERRORED_STATUS) {
|
507 | return console.error('Cancelling versioning data parsing');
|
508 | }
|
509 |
|
510 | proc.pm2_env.vizion_running = false;
|
511 |
|
512 | if (!err) {
|
513 | proc.pm2_env.versioning = meta;
|
514 | proc.pm2_env.versioning.repo_path = current_path;
|
515 | God.notify('online', proc);
|
516 | }
|
517 | else if (err && current_path === last_path) {
|
518 | proc.pm2_env.versioning = null;
|
519 | God.notify('online', proc);
|
520 | }
|
521 | else {
|
522 | last_path = current_path;
|
523 | current_path = path.dirname(current_path);
|
524 | proc.pm2_env.vizion_running = true;
|
525 | vizion.analyze({folder : current_path}, recur_path);
|
526 | }
|
527 | return false;
|
528 | });
|
529 | };
|
530 |
|
531 |
|
532 |
|
533 |
|
534 |
|
535 |
|
536 | God.injectVariables = function injectVariables (env, cb) {
|
537 |
|
538 | var instanceKey = process.env.PM2_PROCESS_INSTANCE_VAR || env.instance_var;
|
539 |
|
540 |
|
541 | var instances = Object.keys(God.clusters_db)
|
542 | .map(function (procId) {
|
543 | return God.clusters_db[procId];
|
544 | }).filter(function (proc) {
|
545 | return proc.pm2_env.name === env.name &&
|
546 | typeof proc.pm2_env[instanceKey] !== 'undefined';
|
547 | }).map(function (proc) {
|
548 | return proc.pm2_env[instanceKey];
|
549 | }).sort(function (a, b) {
|
550 | return b - a;
|
551 | });
|
552 |
|
553 | var instanceNumber = typeof instances[0] === 'undefined' ? 0 : instances[0] + 1;
|
554 |
|
555 | for (var i = 0; i < instances.length; i++) {
|
556 | if (instances.indexOf(i) === -1) {
|
557 | instanceNumber = i;
|
558 | break;
|
559 | }
|
560 | }
|
561 | env[instanceKey] = instanceNumber;
|
562 |
|
563 |
|
564 | if (env.increment_var) {
|
565 | var lastIncrement = Object.keys(God.clusters_db)
|
566 | .map(function (procId) {
|
567 | return God.clusters_db[procId];
|
568 | }).filter(function (proc) {
|
569 | return proc.pm2_env.name === env.name &&
|
570 | typeof proc.pm2_env[env.increment_var] !== 'undefined';
|
571 | }).map(function (proc) {
|
572 | return proc.pm2_env[env.increment_var];
|
573 | }).sort(function (a, b) {
|
574 | return b - a;
|
575 | })[0];
|
576 |
|
577 | var defaut = env.env[env.increment_var] || 0;
|
578 | env[env.increment_var] = typeof lastIncrement === 'undefined' ? defaut : lastIncrement + 1;
|
579 | env.env[env.increment_var] = env[env.increment_var];
|
580 | }
|
581 |
|
582 | return cb(null, env);
|
583 | };
|
584 |
|
585 | God.init()
|