1 | "use strict";
|
2 |
|
3 | var path = require ('path');
|
4 | var fs = require ('fs');
|
5 | var util = require ('util');
|
6 |
|
7 | var EventEmitter = require ('events').EventEmitter;
|
8 |
|
9 | var dataflows = require ('./index');
|
10 | var io = require ('./io/easy');
|
11 | var paint = require ('./color');
|
12 | var common = require ('./common');
|
13 |
|
14 |
|
15 |
|
16 |
|
17 |
|
18 |
|
19 |
|
20 |
|
21 |
|
22 |
|
23 |
|
24 | var Project = function (rootPath) {
|
25 | this.root = new io (rootPath || process.env.PROJECT_ROOT || process.cwd());
|
26 |
|
27 | this.configDir = process.env.PROJECT_CONF || '.dataflows';
|
28 | this.varDir = process.env.PROJECT_VAR || '.dataflows';
|
29 |
|
30 | this.on ('legacy-checked', this.checkConfig.bind(this));
|
31 | this.on ('config-checked', this.readInstance.bind(this));
|
32 | this.on ('instantiated', this.loadConfig.bind(this));
|
33 |
|
34 | this.checkLegacy ();
|
35 |
|
36 |
|
37 |
|
38 |
|
39 |
|
40 |
|
41 | };
|
42 |
|
43 | module.exports = Project;
|
44 |
|
45 | util.inherits (Project, EventEmitter);
|
46 |
|
47 | Project.prototype.checkLegacy = function (cb) {
|
48 | var self = this;
|
49 | this.root.fileIO ('etc/project').stat(function (err, stats) {
|
50 | if (!err && stats && stats.isFile()) {
|
51 | console.error (paint.error ('project has legacy configuration layout. you can migrate by running those commands:'));
|
52 | console.error ("\n\tcd "+self.root.path);
|
53 | console.error ("\tmv etc .dataflows");
|
54 |
|
55 |
|
56 | self.configDir = 'etc';
|
57 | self.varDir = 'var';
|
58 | self.legacy = true;
|
59 | }
|
60 | self.emit ('legacy-checked');
|
61 | });
|
62 | };
|
63 |
|
64 | Project.prototype.checkConfig = function (cb) {
|
65 | var self = this;
|
66 | if (self.legacy) {
|
67 | self.emit ('config-checked');
|
68 | return;
|
69 | }
|
70 |
|
71 |
|
72 | var guessedRoot = this.root;
|
73 | guessedRoot.findUp (this.configDir, function (foundConfigDir) {
|
74 | var detectedRoot = foundConfigDir.parent();
|
75 | if (self.root.path !== detectedRoot.path) {
|
76 | console.log (paint.dataflows (), 'using', paint.path (detectedRoot.path), 'as project root');
|
77 | }
|
78 | self.root = detectedRoot;
|
79 | self.emit ('config-checked');
|
80 | return true;
|
81 | }, function () {
|
82 | self.emit ('error', 'no project config');
|
83 | });
|
84 | };
|
85 |
|
86 |
|
87 | Project.prototype.readInstance = function () {
|
88 | var self = this;
|
89 | this.instance = process.env.PROJECT_INSTANCE;
|
90 | if (this.instance) {
|
91 | console.log (paint.dataflows(), 'instance is:', paint.path (instance));
|
92 | self.emit ('instantiated');
|
93 | return;
|
94 | }
|
95 | var instanceFile = this.root.fileIO (path.join (this.varDir, 'instance'));
|
96 |
|
97 | instanceFile.readFile (function (err, data) {
|
98 |
|
99 | if (err) {
|
100 | var instanceName = [
|
101 | (process.env.USER || process.env.USERNAME),
|
102 | (process.env.HOSTNAME || process.env.COMPUTERNAME)
|
103 | ].join ('@');
|
104 |
|
105 |
|
106 | self.instance = instanceName;
|
107 | self.root.fileIO (path.join (self.varDir, instanceName)).mkdir ();
|
108 | instanceFile.writeFile (instanceName);
|
109 | self.emit ('instantiated');
|
110 | return;
|
111 | }
|
112 |
|
113 |
|
114 |
|
115 |
|
116 |
|
117 |
|
118 |
|
119 | var instance = (""+data).split (/\n/)[0];
|
120 | self.instance = instance == "undefined" ? null : instance;
|
121 | var args = [paint.dataflows(), 'instance is:', paint.path (instance)];
|
122 | if (err) {
|
123 | args.push ('(' + paint.error (err) + ')');
|
124 | } else if (self.legacy) {
|
125 | console.error ("\tmv var/instance .dataflows/");
|
126 | }
|
127 | if (self.legacy) console.log ();
|
128 | console.log.apply (console, args);
|
129 |
|
130 |
|
131 | self.emit ('instantiated');
|
132 | });
|
133 | };
|
134 |
|
135 | Project.prototype.logUnpopulated = function(varPaths) {
|
136 | console.error ("those config variables is unpopulated:");
|
137 | for (var varPath in varPaths) {
|
138 | var value = varPaths[varPath][0];
|
139 | console.log ("\t", paint.path(varPath), '=', value);
|
140 | varPaths[varPath] = value || "<#undefined>";
|
141 | }
|
142 | console.error (
|
143 | "you can run",
|
144 | paint.dataflows ("config set <variable> <value>"),
|
145 | "to define individual variable\nor edit",
|
146 | paint.path (".dataflows/"+this.instance+"/fixup"),
|
147 | "to define all those vars at once"
|
148 | );
|
149 |
|
150 | };
|
151 |
|
152 | Project.prototype.setVariables = function (fixupVars, force) {
|
153 | var self = this;
|
154 |
|
155 | if (!this.instance) {
|
156 | console.log ('Cannot write to the fixup file with undefined instance. Please run', paint.dataflows('init'));
|
157 | process.kill ();
|
158 | }
|
159 |
|
160 | if (!self.fixupConfig)
|
161 | self.fixupConfig = {};
|
162 |
|
163 |
|
164 | Object.keys (fixupVars).forEach (function (varPath) {
|
165 | var pathChunks = [];
|
166 | var root = self.fixupConfig;
|
167 | varPath.split ('.').forEach (function (chunk, index, chunks) {
|
168 | pathChunks[index] = chunk;
|
169 | var newRoot = root[chunk];
|
170 | if (index === chunks.length - 1) {
|
171 | if (force || !(chunk in root)) {
|
172 | root[chunk] = fixupVars[varPath][0] || "<#undefined>";
|
173 | }
|
174 | } else if (!newRoot) {
|
175 | root[chunk] = {};
|
176 | newRoot = root[chunk];
|
177 | }
|
178 | root = newRoot;
|
179 | });
|
180 | });
|
181 |
|
182 |
|
183 | fs.writeFileSync (
|
184 | this.fixupFile,
|
185 | JSON.stringify (this.fixupConfig, null, "\t")
|
186 | );
|
187 | };
|
188 |
|
189 | Project.prototype.formats = [{
|
190 | type: "json",
|
191 | check: /(\/\/\s*json[ \t\n\r]*)?[\{\[]/,
|
192 | parse: function (match, configData) {
|
193 | try {
|
194 | var config = JSON.parse ((""+configData).substr (match[0].length - 1));
|
195 | return {object: config};
|
196 | } catch (e) {
|
197 | return {object: null, error: e};
|
198 | }
|
199 | },
|
200 | stringify: JSON.stringify.bind (JSON),
|
201 | }, {
|
202 | type: "ini",
|
203 | check: /^;\s*ini/,
|
204 | require: "ini",
|
205 | parse: function () {
|
206 |
|
207 | },
|
208 | stringify: function () {
|
209 |
|
210 | }
|
211 | }];
|
212 |
|
213 |
|
214 | Project.prototype.parseConfig = function (configData, configFile) {
|
215 | var self = this;
|
216 | var result;
|
217 | this.formats.some (function (format) {
|
218 | var match = (""+configData).match (format.check);
|
219 | if (match) {
|
220 | result = format.parse (match, configData);
|
221 | result.type = format.type;
|
222 | return true;
|
223 | }
|
224 | });
|
225 | if (!result) {
|
226 | var message =
|
227 | 'Unknown file format in '+(configFile.path || configFile)+'; '
|
228 | + 'for now only JSON supported. You can add new formats using Project.prototype.formats.';
|
229 | console.error (paint.error (message));
|
230 | self.emit ('error', message);
|
231 | }
|
232 | return result;
|
233 | }
|
234 |
|
235 | Project.prototype.interpolateVars = function (error) {
|
236 |
|
237 | var self = this;
|
238 |
|
239 | function iterateNode (node, key, depth) {
|
240 | var value = node[key];
|
241 | var fullKey = depth.join ('.');
|
242 | var match;
|
243 |
|
244 | if (self.variables[fullKey]) {
|
245 | self.variables[fullKey][1] = value;
|
246 | }
|
247 |
|
248 | if ('string' !== typeof value)
|
249 | return;
|
250 |
|
251 | var enchanted = self.isEnchantedValue (value);
|
252 | if (!enchanted) {
|
253 |
|
254 | if (self.variables[fullKey]) {
|
255 | self.variables[fullKey][1] = value.toString ? value.toString() : value;
|
256 | }
|
257 |
|
258 | return;
|
259 | }
|
260 | if ("placeholder" in enchanted) {
|
261 |
|
262 | self.variables[fullKey] = [value];
|
263 | if (enchanted.optional) {
|
264 | self.variables[fullKey][1] = null;
|
265 | node[key] = null;
|
266 | } else if (enchanted.default) {
|
267 | self.variables[fullKey][1] = enchanted.default;
|
268 | node[key] = enchanted.default;
|
269 | }
|
270 | return;
|
271 | }
|
272 | if ("variable" in enchanted) {
|
273 |
|
274 |
|
275 |
|
276 |
|
277 |
|
278 | var varValue = self.getKeyDesc (enchanted.variable.substr (1));
|
279 | if (varValue.enchanted !== undefined) {
|
280 | if ("variable" in varValue.enchanted) {
|
281 | console.error (
|
282 | "variable value cannot contains another variables. used variable",
|
283 | paint.path(enchanted.variable),
|
284 | "which resolves to",
|
285 | paint.path (varValue.value),
|
286 | "in key",
|
287 | paint.path(fullKey)
|
288 | );
|
289 | process.kill ();
|
290 | }
|
291 | self.variables[fullKey] = [value];
|
292 | } else if (varValue.value !== undefined) {
|
293 | node[key] = value.interpolate (self.config, {start: '<', end: '>'});
|
294 | self.variables[fullKey] = [value, node[key]];
|
295 | } else {
|
296 | self.variables[fullKey] = [value];
|
297 | }
|
298 |
|
299 | return;
|
300 | }
|
301 |
|
302 | if ("error" in enchanted || "include" in enchanted) {
|
303 |
|
304 | }
|
305 | }
|
306 |
|
307 | self.iterateTree (self.config, iterateNode, []);
|
308 |
|
309 | var unpopulatedVars = {};
|
310 |
|
311 | var varNames = Object.keys (self.variables);
|
312 | varNames.forEach (function (varName) {
|
313 | if (self.variables[varName][1] !== undefined) {
|
314 |
|
315 | } else {
|
316 | unpopulatedVars[varName] = self.variables[varName];
|
317 | }
|
318 | });
|
319 |
|
320 | this.setVariables (self.variables);
|
321 |
|
322 |
|
323 | if (Object.keys(unpopulatedVars).length || error) {
|
324 | if (unpopulatedVars) {
|
325 | self.logUnpopulated(unpopulatedVars);
|
326 | }
|
327 | self.emit ('error', error || 'unpopulated variables');
|
328 | return;
|
329 | }
|
330 |
|
331 |
|
332 |
|
333 | self.emit ('ready');
|
334 |
|
335 |
|
336 | }
|
337 |
|
338 | Project.prototype.loadConfig = function () {
|
339 |
|
340 | var self = this;
|
341 |
|
342 | var configFile = this.root.fileIO (path.join(this.configDir, 'project'))
|
343 | configFile.readFile (function (err, data) {
|
344 | if (err) {
|
345 | var message = "Can't access "+self.configDir+"/project file. Create one and define project id";
|
346 | console.error (paint.dataflows(), paint.error (message));
|
347 |
|
348 | self.emit ('error', message);
|
349 | return;
|
350 | }
|
351 |
|
352 | var config;
|
353 | var parsed = self.parseConfig (data, configFile);
|
354 | if (parsed.object) {
|
355 | config = parsed.object;
|
356 | } else {
|
357 | var message = 'Project config cannot be parsed:';
|
358 | console.error (message, paint.error (parsed.error));
|
359 | self.emit ('error', message + ' ' + parsed.error.toString());
|
360 | process.kill ();
|
361 | }
|
362 |
|
363 | self.id = config.id;
|
364 |
|
365 |
|
366 | self.loadIncludes(config, 'projectRoot', function (err, config, variables, placeholders) {
|
367 |
|
368 | self.variables = variables;
|
369 | self.placeholders = placeholders;
|
370 |
|
371 | if (err) {
|
372 | console.error (err);
|
373 | console.warn ("Couldn't load includes.");
|
374 |
|
375 | self.interpolateVars();
|
376 | return;
|
377 | }
|
378 |
|
379 | self.config = config;
|
380 |
|
381 | if (!self.instance) {
|
382 | self.interpolateVars ();
|
383 | return;
|
384 | }
|
385 |
|
386 | self.fixupFile = path.join(self.configDir, self.instance, 'fixup');
|
387 |
|
388 | self.root.fileIO (self.fixupFile).readFile (function (err, data) {
|
389 | var fixupConfig = {};
|
390 | if (err) {
|
391 | console.error (
|
392 | "Config fixup file unavailable ("+paint.path (path.join(self.configDir, self.instance, 'fixup'))+")",
|
393 | "Please run", paint.dataflows ('init')
|
394 | );
|
395 | } else {
|
396 | var parsedFixup = self.parseConfig (data, self.fixupFile);
|
397 | if (parsedFixup.object) {
|
398 | self.fixupConfig = fixupConfig = parsedFixup.object;
|
399 | } else {
|
400 | var message = 'Config fixup cannot be parsed:';
|
401 | console.error (message, paint.error (parsedFixup.error));
|
402 | self.emit ('error', message + ' ' + parsedFixup.error.toString());
|
403 | process.kill ();
|
404 | }
|
405 | }
|
406 |
|
407 | util.extend (true, self.config, fixupConfig);
|
408 |
|
409 | self.interpolateVars ();
|
410 |
|
411 | });
|
412 | });
|
413 | });
|
414 | };
|
415 |
|
416 | function Config () {
|
417 |
|
418 | }
|
419 |
|
420 | Config.prototype.getValueByKey = function (key) {
|
421 |
|
422 | var value = common.getByPath (key, this);
|
423 | if (this.isEnchanted (value)) {
|
424 | return null;
|
425 | }
|
426 | return value;
|
427 | }
|
428 |
|
429 | Project.prototype.connectors = {};
|
430 | Project.prototype.connections = {};
|
431 |
|
432 | Project.prototype.getModule = function (type, name, optional) {
|
433 | var self = this;
|
434 | optional = optional || false;
|
435 | var mod;
|
436 | var taskFound = [
|
437 | path.join('dataflo.ws', type, name),
|
438 | path.resolve(this.root.path, type, name),
|
439 | path.resolve(this.root.path, 'node_modules', type, name),
|
440 | name
|
441 | ].some (function (modPath) {
|
442 | try {
|
443 | mod = require(modPath);
|
444 | return true;
|
445 | } catch (e) {
|
446 |
|
447 | if (e.toString().indexOf(name + '\'') > 0 && e.code == "MODULE_NOT_FOUND") {
|
448 | return false;
|
449 | } else {
|
450 | console.error ('requirement failed:', paint.error (e.toString()), "in", paint.path (self.root.relative (modPath)));
|
451 | return true;
|
452 | }
|
453 | }
|
454 | });
|
455 |
|
456 | if (!mod && !optional)
|
457 | console.error ("module " + type + " " + name + " cannot be used");
|
458 |
|
459 | return mod;
|
460 | };
|
461 |
|
462 | Project.prototype.getInitiator = function (name) {
|
463 | return this.getModule('initiator', name);
|
464 | };
|
465 |
|
466 | Project.prototype.getTask = function (name) {
|
467 | return this.getModule('task', name);
|
468 | };
|
469 |
|
470 | Project.prototype.require = function (name, optional) {
|
471 | return this.getModule('', name, optional);
|
472 | };
|
473 |
|
474 | var configCache = {};
|
475 |
|
476 | Project.prototype.iterateTree = function iterateTree (tree, cb, depth) {
|
477 | if (null == tree)
|
478 | return;
|
479 |
|
480 | var level = depth.length;
|
481 |
|
482 | var step = function (node, key, tree) {
|
483 | depth[level] = key;
|
484 | cb (tree, key, depth);
|
485 | iterateTree (node, cb, depth.slice (0));
|
486 | };
|
487 |
|
488 | if (Array === tree.constructor) {
|
489 | tree.forEach (step);
|
490 | } else if (Object === tree.constructor) {
|
491 | Object.keys(tree).forEach(function (key) {
|
492 | step (tree[key], key, tree);
|
493 | });
|
494 | }
|
495 | };
|
496 |
|
497 | Project.prototype.getKeyDesc = function (key) {
|
498 | var result = {};
|
499 | var value = common.getByPath (key, this.config);
|
500 | result.value = value.value;
|
501 | result.enchanted = this.isEnchantedValue (result.value);
|
502 |
|
503 | if (result.enchanted && "variable" in result.enchanted) {
|
504 | result.interpolated = result.value.interpolate();
|
505 | return result;
|
506 | }
|
507 | return result;
|
508 | }
|
509 |
|
510 |
|
511 | Project.prototype.getValue = function (key) {
|
512 | var value = common.getByPath (key, this.config).value;
|
513 | if (value === undefined)
|
514 | return;
|
515 | var enchanted = this.isEnchantedValue (value);
|
516 |
|
517 | if (enchanted && "variable" in enchanted) {
|
518 | var result = new String (value.interpolate());
|
519 | result.rawValue = value;
|
520 | return result;
|
521 | }
|
522 | return value;
|
523 | }
|
524 |
|
525 | Project.prototype.isEnchantedValue = function (value) {
|
526 |
|
527 | var tagRe = /<(([\$\#]*)((optional|default):)?([^>]+))>/;
|
528 | var result;
|
529 |
|
530 | if ('string' !== typeof value) {
|
531 | return;
|
532 | }
|
533 | var check = value.match (tagRe);
|
534 | if (check) {
|
535 | if (check[2] === "$") {
|
536 | return {"variable": check[1]};
|
537 | } else if (check[2] === "#") {
|
538 | result = {"placeholder": check[1]};
|
539 | if (check[4]) result[check[4]] = check[5];
|
540 | return result;
|
541 | } else if (check[0].length === value.length) {
|
542 | return {"include": check[1]};
|
543 | } else {
|
544 | return {"error": true};
|
545 | }
|
546 | }
|
547 | }
|
548 |
|
549 |
|
550 | Project.prototype.loadIncludes = function (config, level, cb) {
|
551 | var self = this;
|
552 |
|
553 | var DEFAULT_ROOT = this.configDir,
|
554 | DELIMITER = ' > ',
|
555 | cnt = 0,
|
556 | len = 0;
|
557 |
|
558 | var levelHash = {};
|
559 |
|
560 | var variables = {};
|
561 | var placeholders = {};
|
562 |
|
563 | level.split(DELIMITER).forEach(function(key) {
|
564 | levelHash[key] = true;
|
565 | });
|
566 |
|
567 | function onLoad() {
|
568 | cnt += 1;
|
569 | if (cnt >= len) {
|
570 | cb(null, config, variables, placeholders);
|
571 | }
|
572 | }
|
573 |
|
574 | function onError(err) {
|
575 | console.log('[WARNING] Level:', level, 'is not correct.\nError:', paint.error (err));
|
576 | cb(err, config, variables, placeholders);
|
577 | }
|
578 |
|
579 | function iterateNode (node, key, depth) {
|
580 | var value = node[key];
|
581 |
|
582 | if ('string' !== typeof value)
|
583 | return;
|
584 |
|
585 | var enchanted = self.isEnchantedValue (value);
|
586 | if (!enchanted)
|
587 | return;
|
588 | if ("variable" in enchanted) {
|
589 | variables[depth.join ('.')] = [value];
|
590 | return;
|
591 | }
|
592 | if ("placeholder" in enchanted) {
|
593 | variables[depth.join ('.')] = [value];
|
594 | return;
|
595 | }
|
596 | if ("error" in enchanted) {
|
597 | console.error ('bad include tag:', "\"" + value + "\"");
|
598 | onError();
|
599 | return;
|
600 | }
|
601 | if ("include" in enchanted) {
|
602 | len ++;
|
603 | var incPath = enchanted.include;
|
604 |
|
605 | if (0 !== incPath.indexOf('/')) {
|
606 | incPath = path.join (DEFAULT_ROOT, incPath);
|
607 | }
|
608 |
|
609 | if (incPath in levelHash) {
|
610 |
|
611 | throw new Error('circular linking');
|
612 | }
|
613 |
|
614 | delete node[key];
|
615 |
|
616 | if (configCache[incPath]) {
|
617 |
|
618 | node[key] = util.clone(configCache[incPath]);
|
619 | onLoad();
|
620 | return;
|
621 |
|
622 | }
|
623 |
|
624 | self.root.fileIO(incPath).readFile(function (err, data) {
|
625 | if (err) {
|
626 | onError(err);
|
627 | return;
|
628 | }
|
629 |
|
630 | self.loadIncludes(JSON.parse(data), path.join(level, DELIMITER, incPath), function(tree, includeConfig) {
|
631 | configCache[incPath] = includeConfig;
|
632 |
|
633 | node[key] = util.clone(configCache[incPath]);
|
634 | onLoad();
|
635 | });
|
636 | });
|
637 |
|
638 | }
|
639 | }
|
640 |
|
641 | this.iterateTree(config, iterateNode, []);
|
642 |
|
643 |
|
644 |
|
645 | !len && cb(null, config, variables, placeholders);
|
646 | };
|