1 | "use strict";
|
2 |
|
3 | var path = require ('path');
|
4 | var fs = require ('fs');
|
5 | var util = require ('util');
|
6 |
|
7 | var EventEmitter = require ('events').EventEmitter;
|
8 |
|
9 | var dataflows = require ('./');
|
10 | var io = require ('./io/easy');
|
11 | var paint = dataflows.color;
|
12 | var common = dataflows.common;
|
13 |
|
14 |
|
15 |
|
16 |
|
17 |
|
18 |
|
19 |
|
20 |
|
21 |
|
22 |
|
23 |
|
24 | var Project = function (rootPath) {
|
25 | this.root = new io (rootPath || process.env.PROJECT_ROOT || process.cwd());
|
26 |
|
27 | this.configDir = process.env.PROJECT_CONF || '.dataflows';
|
28 | this.varDir = process.env.PROJECT_VAR || '.dataflows';
|
29 |
|
30 | this.on ('legacy-checked', this.checkConfig.bind(this));
|
31 | this.on ('config-checked', this.readInstance.bind(this));
|
32 | this.on ('instantiated', this.findAndLoad.bind(this, "project"));
|
33 |
|
34 | this.checkLegacy ();
|
35 |
|
36 |
|
37 |
|
38 |
|
39 |
|
40 |
|
41 | };
|
42 |
|
43 | module.exports = Project;
|
44 |
|
45 | util.inherits (Project, EventEmitter);
|
46 |
|
47 | Project.prototype.checkLegacy = function (cb) {
|
48 | var self = this;
|
49 | this.root.fileIO ('etc/project').stat(function (err, stats) {
|
50 | if (!err && stats && stats.isFile()) {
|
51 | console.error (paint.error ('project has legacy configuration layout. you can migrate by running those commands:'));
|
52 | console.error ("\n\tcd "+self.root.path);
|
53 | console.error ("\tmv etc .dataflows");
|
54 |
|
55 |
|
56 | self.configDir = 'etc';
|
57 | self.varDir = 'var';
|
58 | self.legacy = true;
|
59 | }
|
60 | self.emit ('legacy-checked');
|
61 | });
|
62 | };
|
63 |
|
64 | Project.prototype.checkConfig = function (cb) {
|
65 | var self = this;
|
66 | if (self.legacy) {
|
67 | self.emit ('config-checked');
|
68 | return;
|
69 | }
|
70 |
|
71 |
|
72 | var guessedRoot = this.root;
|
73 | guessedRoot.findUp (this.configDir, function (foundConfigDir) {
|
74 | var detectedRoot = foundConfigDir.parent();
|
75 | if (self.root.path !== detectedRoot.path) {
|
76 | console.log (paint.dataflows (), 'using', paint.path (detectedRoot.path), 'as project root');
|
77 | }
|
78 | self.root = detectedRoot;
|
79 | self.emit ('config-checked');
|
80 | return true;
|
81 | }, function () {
|
82 | self.emit ('error', 'no project config');
|
83 | });
|
84 | };
|
85 |
|
86 |
|
87 | Project.prototype.readInstance = function () {
|
88 | var self = this;
|
89 | this.instance = process.env.PROJECT_INSTANCE;
|
90 | if (this.instance) {
|
91 | console.log (paint.dataflows(), 'instance is:', paint.path (instance));
|
92 | self.emit ('instantiated');
|
93 | return;
|
94 | }
|
95 | var instanceFile = this.root.fileIO (path.join (this.varDir, 'instance'));
|
96 |
|
97 | instanceFile.readFile (function (err, data) {
|
98 |
|
99 | if (err) {
|
100 | var instanceName = [
|
101 | (process.env.USER || process.env.USERNAME),
|
102 | (process.env.HOSTNAME || process.env.COMPUTERNAME)
|
103 | ].join ('@');
|
104 |
|
105 |
|
106 | self.instance = instanceName;
|
107 | self.root.fileIO (path.join (self.varDir, instanceName)).mkdir ();
|
108 | instanceFile.writeFile (instanceName);
|
109 | self.emit ('instantiated');
|
110 | return;
|
111 | }
|
112 |
|
113 |
|
114 |
|
115 |
|
116 |
|
117 |
|
118 |
|
119 | var instance = (""+data).split (/\n/)[0];
|
120 | self.instance = instance == "undefined" ? null : instance;
|
121 | var args = [paint.dataflows(), 'instance is:', paint.path (instance)];
|
122 | if (err) {
|
123 | args.push ('(' + paint.error (err) + ')');
|
124 | } else if (self.legacy) {
|
125 | console.error ("\tmv var/instance .dataflows/");
|
126 | }
|
127 | if (self.legacy) console.log ();
|
128 | console.log.apply (console, args);
|
129 |
|
130 |
|
131 | self.emit ('instantiated');
|
132 | });
|
133 | };
|
134 |
|
135 | Project.prototype.logUnpopulated = function(varPaths) {
|
136 | console.error ("those config variables is unpopulated:");
|
137 | for (var varPath in varPaths) {
|
138 | var value = varPaths[varPath][0];
|
139 | console.log ("\t", paint.path(varPath), '=', value);
|
140 | varPaths[varPath] = value || "<#undefined>";
|
141 | }
|
142 | console.error (
|
143 | "you can run",
|
144 | paint.dataflows ("config set <variable> <value>"),
|
145 | "to define individual variable\nor edit",
|
146 | paint.path (".dataflows/"+this.instance+"/"+path.basename (this.fixupFile)),
|
147 | "to define all those vars at once"
|
148 | );
|
149 |
|
150 | };
|
151 |
|
152 | Project.prototype.setVariables = function (fixupVars, force) {
|
153 | var self = this;
|
154 |
|
155 | if (!this.instance) {
|
156 | console.log ('Cannot write to the fixup file with undefined instance. Please run', paint.dataflows('init'));
|
157 | process.kill ();
|
158 | }
|
159 |
|
160 | if (!self.fixupConfig)
|
161 | self.fixupConfig = {};
|
162 |
|
163 |
|
164 | Object.keys (fixupVars).forEach (function (varPath) {
|
165 | var pathChunks = [];
|
166 | var root = self.fixupConfig;
|
167 | varPath.split ('.').forEach (function (chunk, index, chunks) {
|
168 | pathChunks[index] = chunk;
|
169 | var newRoot = root[chunk];
|
170 | if (index === chunks.length - 1) {
|
171 | if (force || !(chunk in root)) {
|
172 | root[chunk] = fixupVars[varPath][0] || "<#undefined>";
|
173 | }
|
174 | } else if (!newRoot) {
|
175 | root[chunk] = {};
|
176 | newRoot = root[chunk];
|
177 | }
|
178 | root = newRoot;
|
179 | });
|
180 | });
|
181 |
|
182 |
|
183 | fs.writeFileSync (
|
184 | this.fixupFile,
|
185 | JSON.stringify (this.fixupConfig, null, "\t")
|
186 | );
|
187 | };
|
188 |
|
189 | Project.prototype.formats = [{
|
190 | type: "json",
|
191 | check: /(\/\/\s*json[ \t\n\r]*)?[\{\[]/,
|
192 | parse: function (match, configData) {
|
193 | try {
|
194 | var config = JSON.parse ((""+configData).substr (match[0].length - 1));
|
195 | return {object: config};
|
196 | } catch (e) {
|
197 | return {object: null, error: e};
|
198 | }
|
199 | },
|
200 | stringify: JSON.stringify.bind (JSON),
|
201 | }, {
|
202 | type: "ini",
|
203 | check: /^;\s*ini/,
|
204 | require: "ini",
|
205 | parse: function () {
|
206 |
|
207 | },
|
208 | stringify: function () {
|
209 |
|
210 | }
|
211 | }];
|
212 |
|
213 |
|
214 | Project.prototype.parseConfig = function (configData, configFile) {
|
215 | var self = this;
|
216 | var result;
|
217 | this.formats.some (function (format) {
|
218 | var match = (""+configData).match (format.check);
|
219 | if (match) {
|
220 | result = format.parse (match, configData);
|
221 | result.type = format.type;
|
222 | return true;
|
223 | }
|
224 | });
|
225 | if (!result) {
|
226 | var message =
|
227 | 'Unknown file format in '+(configFile.path || configFile)+'; '
|
228 | + 'for now only JSON supported. You can add new formats using Project.prototype.formats.';
|
229 | console.error (paint.error (message));
|
230 | self.emit ('error', message);
|
231 | }
|
232 | return result;
|
233 | }
|
234 |
|
235 | Project.prototype.interpolateVars = function (error) {
|
236 |
|
237 | var self = this;
|
238 |
|
239 | function iterateNode (node, key, depth) {
|
240 | var value = node[key];
|
241 | var fullKey = depth.join ('.');
|
242 | var match;
|
243 |
|
244 | if (self.variables[fullKey]) {
|
245 | self.variables[fullKey][1] = value;
|
246 | }
|
247 |
|
248 | if ('string' !== typeof value)
|
249 | return;
|
250 |
|
251 | var enchanted = self.isEnchantedValue (value);
|
252 | if (!enchanted) {
|
253 |
|
254 | if (self.variables[fullKey]) {
|
255 | self.variables[fullKey][1] = value.toString ? value.toString() : value;
|
256 | }
|
257 |
|
258 | return;
|
259 | }
|
260 | if ("placeholder" in enchanted) {
|
261 |
|
262 | self.variables[fullKey] = [value];
|
263 | if (enchanted.optional) {
|
264 | self.variables[fullKey][1] = null;
|
265 | node[key] = null;
|
266 | } else if (enchanted.default) {
|
267 | self.variables[fullKey][1] = enchanted.default;
|
268 | node[key] = enchanted.default;
|
269 | }
|
270 | return;
|
271 | }
|
272 | if ("variable" in enchanted) {
|
273 |
|
274 |
|
275 |
|
276 |
|
277 |
|
278 | var varValue = self.getKeyDesc (enchanted.variable.substr (1));
|
279 | if (varValue.enchanted !== undefined) {
|
280 | if ("variable" in varValue.enchanted) {
|
281 | console.error (
|
282 | "variable value cannot contains another variables. used variable",
|
283 | paint.path(enchanted.variable),
|
284 | "which resolves to",
|
285 | paint.path (varValue.value),
|
286 | "in key",
|
287 | paint.path(fullKey)
|
288 | );
|
289 | process.kill ();
|
290 | }
|
291 | self.variables[fullKey] = [value];
|
292 | } else if (varValue.value !== undefined) {
|
293 | node[key] = value.interpolate (self.config, {start: '<', end: '>'});
|
294 | self.variables[fullKey] = [value, node[key]];
|
295 | } else {
|
296 | self.variables[fullKey] = [value];
|
297 | }
|
298 |
|
299 | return;
|
300 | }
|
301 |
|
302 | if ("error" in enchanted || "include" in enchanted) {
|
303 |
|
304 | }
|
305 | }
|
306 |
|
307 | self.iterateTree (self.config, iterateNode, []);
|
308 |
|
309 | var unpopulatedVars = {};
|
310 |
|
311 | var varNames = Object.keys (self.variables);
|
312 | varNames.forEach (function (varName) {
|
313 | if (self.variables[varName][1] !== undefined) {
|
314 |
|
315 | } else {
|
316 | unpopulatedVars[varName] = self.variables[varName];
|
317 | }
|
318 | });
|
319 |
|
320 | this.setVariables (self.variables);
|
321 |
|
322 |
|
323 | if (Object.keys(unpopulatedVars).length || error) {
|
324 | if (unpopulatedVars) {
|
325 | self.logUnpopulated(unpopulatedVars);
|
326 | }
|
327 | self.emit ('error', error || 'unpopulated variables');
|
328 | return;
|
329 | }
|
330 |
|
331 |
|
332 |
|
333 | self.emit ('ready');
|
334 |
|
335 |
|
336 | }
|
337 |
|
338 |
|
339 |
|
340 |
|
341 |
|
342 | Project.prototype.findAndLoad = function (type, cb) {
|
343 |
|
344 | var dirToRead;
|
345 | if (type === "project") {
|
346 | dirToRead = this.configDir;
|
347 | } else {
|
348 | dirToRead = path.join (this.configDir, this.instance);
|
349 | }
|
350 |
|
351 | fs.readdir (dirToRead, function (err, files) {
|
352 | var configFileName;
|
353 | files.some (function (fileName) {
|
354 | if (path.basename (fileName, path.extname (fileName)) === type) {
|
355 | configFileName = fileName;
|
356 | return true;
|
357 | }
|
358 | });
|
359 |
|
360 |
|
361 |
|
362 | if (configFileName && type === "project") {
|
363 | this.loadConfig (new io (path.join (dirToRead, configFileName)));
|
364 | return;
|
365 | } else if (type === "fixup" && cb) {
|
366 | cb (new io (path.join (dirToRead, configFileName || "fixup.json")));
|
367 | return;
|
368 | }
|
369 |
|
370 | var message = "Can't find "+type+" config in " + this.configDir + " folder. Please relaunch `dataflows init`";
|
371 | console.error (paint.dataflows(), paint.error (message));
|
372 |
|
373 | this.emit ('error', message);
|
374 |
|
375 | }.bind (this));
|
376 |
|
377 | }
|
378 |
|
379 | Project.prototype.loadConfig = function (configFile) {
|
380 |
|
381 | var self = this;
|
382 |
|
383 |
|
384 | configFile.readFile (function (err, data) {
|
385 | if (err) {
|
386 | var message = "Can't access "+configFile.path+" file. Create one and define project id";
|
387 | console.error (paint.dataflows(), paint.error (message));
|
388 |
|
389 | self.emit ('error', message);
|
390 | return;
|
391 | }
|
392 |
|
393 | var config;
|
394 | var parsed = self.parseConfig (data, configFile);
|
395 | if (parsed.object) {
|
396 | config = parsed.object;
|
397 | } else {
|
398 | var message = 'Project config cannot be parsed:';
|
399 | console.error (message, paint.error (parsed.error));
|
400 | self.emit ('error', message + ' ' + parsed.error.toString());
|
401 | process.kill ();
|
402 | }
|
403 |
|
404 | self.id = config.id;
|
405 |
|
406 |
|
407 | self.loadIncludes(config, 'projectRoot', function (err, config, variables, placeholders) {
|
408 |
|
409 | self.variables = variables;
|
410 | self.placeholders = placeholders;
|
411 |
|
412 | if (err) {
|
413 | console.error (err);
|
414 | console.warn ("Couldn't load includes.");
|
415 |
|
416 | self.interpolateVars();
|
417 | return;
|
418 | }
|
419 |
|
420 | self.config = config;
|
421 |
|
422 | if (!self.instance) {
|
423 | self.interpolateVars ();
|
424 | return;
|
425 | }
|
426 |
|
427 | self.findAndLoad ("fixup", function (fixupFile) {
|
428 | self.fixupFile = fixupFile.path;
|
429 | fixupFile.readFile (function (err, data) {
|
430 | var fixupConfig = {};
|
431 | if (err) {
|
432 | console.error (
|
433 | "Config fixup file unavailable ("+paint.path (self.fixupFile)+")",
|
434 | "Please run", paint.dataflows ('init')
|
435 | );
|
436 | } else {
|
437 | var parsedFixup = self.parseConfig (data, self.fixupFile);
|
438 | if (parsedFixup.object) {
|
439 | self.fixupConfig = fixupConfig = parsedFixup.object;
|
440 | } else {
|
441 | var message = 'Config fixup cannot be parsed:';
|
442 | console.error (message, paint.error (parsedFixup.error));
|
443 | self.emit ('error', message + ' ' + parsedFixup.error.toString());
|
444 | process.kill ();
|
445 | }
|
446 | }
|
447 |
|
448 | util.extend (true, self.config, fixupConfig);
|
449 |
|
450 | self.interpolateVars ();
|
451 |
|
452 | });
|
453 | });
|
454 | });
|
455 | });
|
456 | };
|
457 |
|
458 | function Config () {
|
459 |
|
460 | }
|
461 |
|
462 | Config.prototype.getValueByKey = function (key) {
|
463 |
|
464 | var value = common.getByPath (key, this);
|
465 | if (this.isEnchanted (value)) {
|
466 | return null;
|
467 | }
|
468 | return value;
|
469 | }
|
470 |
|
471 | Project.prototype.connectors = {};
|
472 | Project.prototype.connections = {};
|
473 |
|
474 | Project.prototype.getModule = function (type, name, optional) {
|
475 | var self = this;
|
476 | optional = optional || false;
|
477 | var mod;
|
478 | var taskFound = [
|
479 | path.join('dataflo.ws', type, name),
|
480 | path.resolve(this.root.path, type, name),
|
481 | path.resolve(this.root.path, 'node_modules', type, name),
|
482 | name
|
483 | ].some (function (modPath) {
|
484 | try {
|
485 | mod = require(modPath);
|
486 | return true;
|
487 | } catch (e) {
|
488 |
|
489 | if (e.toString().indexOf(name + '\'') > 0 && e.code == "MODULE_NOT_FOUND") {
|
490 | return false;
|
491 | } else {
|
492 | console.error ('requirement failed:', paint.error (e.toString()), "in", paint.path (self.root.relative (modPath)));
|
493 | return true;
|
494 | }
|
495 | }
|
496 | });
|
497 |
|
498 | if (!mod && !optional)
|
499 | console.error ("module " + type + " " + name + " cannot be used");
|
500 |
|
501 | return mod;
|
502 | };
|
503 |
|
504 | Project.prototype.getInitiator = function (name) {
|
505 | return this.getModule('initiator', name);
|
506 | };
|
507 |
|
508 | Project.prototype.getTask = function (name) {
|
509 | return this.getModule('task', name);
|
510 | };
|
511 |
|
512 | Project.prototype.require = function (name, optional) {
|
513 | return this.getModule('', name, optional);
|
514 | };
|
515 |
|
516 | var configCache = {};
|
517 |
|
518 | Project.prototype.iterateTree = function iterateTree (tree, cb, depth) {
|
519 | if (null == tree)
|
520 | return;
|
521 |
|
522 | var level = depth.length;
|
523 |
|
524 | var step = function (node, key, tree) {
|
525 | depth[level] = key;
|
526 | cb (tree, key, depth);
|
527 | iterateTree (node, cb, depth.slice (0));
|
528 | };
|
529 |
|
530 | if (Array === tree.constructor) {
|
531 | tree.forEach (step);
|
532 | } else if (Object === tree.constructor) {
|
533 | Object.keys(tree).forEach(function (key) {
|
534 | step (tree[key], key, tree);
|
535 | });
|
536 | }
|
537 | };
|
538 |
|
539 | Project.prototype.getKeyDesc = function (key) {
|
540 | var result = {};
|
541 | var value = common.getByPath (key, this.config);
|
542 | result.value = value.value;
|
543 | result.enchanted = this.isEnchantedValue (result.value);
|
544 |
|
545 | if (result.enchanted && "variable" in result.enchanted) {
|
546 | result.interpolated = result.value.interpolate();
|
547 | return result;
|
548 | }
|
549 | return result;
|
550 | }
|
551 |
|
552 |
|
553 | Project.prototype.getValue = function (key) {
|
554 | var value = common.getByPath (key, this.config).value;
|
555 | if (value === undefined)
|
556 | return;
|
557 | var enchanted = this.isEnchantedValue (value);
|
558 |
|
559 | if (enchanted && "variable" in enchanted) {
|
560 | var result = new String (value.interpolate());
|
561 | result.rawValue = value;
|
562 | return result;
|
563 | }
|
564 | return value;
|
565 | }
|
566 |
|
567 | Project.prototype.isEnchantedValue = function (value) {
|
568 |
|
569 | var tagRe = /<(([\$\#]*)((optional|default):)?([^>]+))>/;
|
570 | var result;
|
571 |
|
572 | if ('string' !== typeof value) {
|
573 | return;
|
574 | }
|
575 | var check = value.match (tagRe);
|
576 | if (check) {
|
577 | if (check[2] === "$") {
|
578 | return {"variable": check[1]};
|
579 | } else if (check[2] === "#") {
|
580 | result = {"placeholder": check[1]};
|
581 | if (check[4]) result[check[4]] = check[5];
|
582 | return result;
|
583 | } else if (check[0].length === value.length) {
|
584 | return {"include": check[1]};
|
585 | } else {
|
586 | return {"error": true};
|
587 | }
|
588 | }
|
589 | }
|
590 |
|
591 |
|
592 | Project.prototype.loadIncludes = function (config, level, cb) {
|
593 | var self = this;
|
594 |
|
595 | var DEFAULT_ROOT = this.configDir,
|
596 | DELIMITER = ' > ',
|
597 | cnt = 0,
|
598 | len = 0;
|
599 |
|
600 | var levelHash = {};
|
601 |
|
602 | var variables = {};
|
603 | var placeholders = {};
|
604 |
|
605 | level.split(DELIMITER).forEach(function(key) {
|
606 | levelHash[key] = true;
|
607 | });
|
608 |
|
609 | function onLoad() {
|
610 | cnt += 1;
|
611 | if (cnt >= len) {
|
612 | cb(null, config, variables, placeholders);
|
613 | }
|
614 | }
|
615 |
|
616 | function onError(err) {
|
617 | console.log('[WARNING] Level:', level, 'is not correct.\nError:', paint.error (err));
|
618 | cb(err, config, variables, placeholders);
|
619 | }
|
620 |
|
621 | function iterateNode (node, key, depth) {
|
622 | var value = node[key];
|
623 |
|
624 | if ('string' !== typeof value)
|
625 | return;
|
626 |
|
627 | var enchanted = self.isEnchantedValue (value);
|
628 | if (!enchanted)
|
629 | return;
|
630 | if ("variable" in enchanted) {
|
631 | variables[depth.join ('.')] = [value];
|
632 | return;
|
633 | }
|
634 | if ("placeholder" in enchanted) {
|
635 | variables[depth.join ('.')] = [value];
|
636 | return;
|
637 | }
|
638 | if ("error" in enchanted) {
|
639 | console.error ('bad include tag:', "\"" + value + "\"");
|
640 | onError();
|
641 | return;
|
642 | }
|
643 | if ("include" in enchanted) {
|
644 | len ++;
|
645 | var incPath = enchanted.include;
|
646 |
|
647 | if (0 !== incPath.indexOf('/')) {
|
648 | incPath = path.join (DEFAULT_ROOT, incPath);
|
649 | }
|
650 |
|
651 | if (incPath in levelHash) {
|
652 |
|
653 | throw new Error('circular linking');
|
654 | }
|
655 |
|
656 | delete node[key];
|
657 |
|
658 | if (configCache[incPath]) {
|
659 |
|
660 | node[key] = util.clone(configCache[incPath]);
|
661 | onLoad();
|
662 | return;
|
663 |
|
664 | }
|
665 |
|
666 | self.root.fileIO(incPath).readFile(function (err, data) {
|
667 | if (err) {
|
668 | onError(err);
|
669 | return;
|
670 | }
|
671 |
|
672 | self.loadIncludes(JSON.parse(data), path.join(level, DELIMITER, incPath), function(tree, includeConfig) {
|
673 | configCache[incPath] = includeConfig;
|
674 |
|
675 | node[key] = util.clone(configCache[incPath]);
|
676 | onLoad();
|
677 | });
|
678 | });
|
679 |
|
680 | }
|
681 | }
|
682 |
|
683 | this.iterateTree(config, iterateNode, []);
|
684 |
|
685 |
|
686 |
|
687 | !len && cb(null, config, variables, placeholders);
|
688 | };
|