1 | /**
|
2 | * FsTools
|
3 | *
|
4 | * Collection of FS related tools, that stdlib lack of.
|
5 | **/
|
6 |
|
7 |
|
8 | ;
|
9 |
|
10 |
|
11 | // Node < 0.8 shims
|
12 | if (!require('fs').exists) {
|
13 | require('fs').exists = require('path').exists;
|
14 | }
|
15 | if (!require('fs').existsSync) {
|
16 | require('fs').existsSync = require('path').existsSync;
|
17 | }
|
18 |
|
19 |
|
20 | // stdlib
|
21 | var fs = require('fs');
|
22 | var path_join = require('path').join;
|
23 | var path_normalize = require('path').normalize;
|
24 | var dirname = require('path').dirname;
|
25 | var crypto = require('crypto');
|
26 |
|
27 |
|
28 | // 3rd-party
|
29 | var async = require('async');
|
30 | var _ = require('underscore');
|
31 |
|
32 |
|
33 | // epxorts: walk, mkdir, copy, remove
|
34 | var fstools = module.exports = {};
|
35 |
|
36 |
|
37 | // INTERNAL HELPERS
|
38 | ////////////////////////////////////////////////////////////////////////////////
|
39 |
|
40 |
|
41 | // walk_flat(path, iterator, callback) -> void
|
42 | // - path (String): Path to iterate through
|
43 | // - iterator (Function): Will be fired on each element within `path`
|
44 | // - callback (Function): Will be fired once all files were processed
|
45 | //
|
46 | // Walks through given `path` and calls `iterator(path, callback)` for
|
47 | // each found entry (regardless to it's type) and waits for all `callback`s
|
48 | // (passed to iterator) to be fired. After all callbacks were fired, fires
|
49 | // `callback` (given to walk_flat itself).
|
50 | //
|
51 | // NOTICE: It walks through single dimension of file system - it won't go into
|
52 | // found sub-directories. See `walk_deep` for this puprpose.
|
53 | //
|
54 | // Example:
|
55 | //
|
56 | // walk_flat('/home/nodeca', function (path, callback) {
|
57 | // if ('/home/nodeca/secrets.yml' === path) {
|
58 | // callback(Error("There is secrets file."));
|
59 | // return;
|
60 | // }
|
61 | //
|
62 | // if ('/home/nodeca/xxx' === path) {
|
63 | // callback(Error("Path contains some porno?"));
|
64 | // return;
|
65 | // }
|
66 | //
|
67 | // callback();
|
68 | // }, function (err) {
|
69 | // if (err) console.error(err);
|
70 | // console.log('Done');
|
71 | // });
|
72 | //
|
73 | function walk_flat(path, iterator, callback) {
|
74 | fs.readdir(path, function (err, files) {
|
75 | if (err) {
|
76 | if (err && 'ENOENT' === err.code) {
|
77 | callback(null);
|
78 | return;
|
79 | }
|
80 |
|
81 | callback(err);
|
82 | return;
|
83 | }
|
84 |
|
85 | async.forEach(files, function (file, next) {
|
86 | iterator(path_join(path, file), next);
|
87 | }, callback);
|
88 | });
|
89 | }
|
90 |
|
91 |
|
92 | // walk_deep(path, match, iterator, callback) -> void
|
93 | // - path (String): Path to iterate through
|
94 | // - match (Function): Path test whenever iterator should be executed or not
|
95 | // - iterator (Function): Will be fired on each element within `path`
|
96 | // - callback (Function): Will be fired once all files were processed
|
97 | //
|
98 | // Walks through given `path` with all it's nested childs calling
|
99 | // `iterator(path, callback)` for each found file.
|
100 | //
|
101 | function walk_deep(path, match, iterator, callback) {
|
102 | walk_flat(path_normalize(path), function (path, next) {
|
103 | fs.lstat(path, function (err, stats) {
|
104 | if (err) {
|
105 | next(err);
|
106 | return;
|
107 | }
|
108 |
|
109 | if (stats.isDirectory()) {
|
110 | walk_deep(path, match, iterator, next);
|
111 | return;
|
112 | }
|
113 |
|
114 | if (match(path)) {
|
115 | iterator(path, stats, next);
|
116 | return;
|
117 | }
|
118 |
|
119 | next();
|
120 | });
|
121 | }, callback);
|
122 | }
|
123 |
|
124 |
|
125 | function copy_file(src, dst, callback) {
|
126 | var ifd, ofd;
|
127 |
|
128 | // create streams
|
129 | ifd = fs.createReadStream(src, {bufferSize: 64 * 1024}).on('error', callback);
|
130 | ofd = fs.createWriteStream(dst).on('error', callback).on('close', callback);
|
131 |
|
132 | // pipe src to dst
|
133 | ifd.pipe(ofd);
|
134 | }
|
135 |
|
136 |
|
137 | // PUBLIC API
|
138 | ////////////////////////////////////////////////////////////////////////////////
|
139 |
|
140 |
|
141 | /**
|
142 | * FsTools.walk(path, pattern, iterator, callback) -> void
|
143 | * FsTools.walk(path, iterator, callback) -> void
|
144 | *
|
145 | * Walks throught all files withing `path` (including sub-dirs) and calls
|
146 | * `iterator` on each found file (or block device etc.) matching `pattern`.
|
147 | * If no `pattern` was given - will fire call `iterator` for every single
|
148 | * path found. After all iterations will call `callback` (if it was specified)
|
149 | * with passing `error` as first arguemtn if there was an error.
|
150 | *
|
151 | * If `path` points a file, iterator will be called against it (respecting
|
152 | * pattern if it was given).
|
153 | *
|
154 | *
|
155 | * ##### Iterator
|
156 | *
|
157 | * All iterations are running within promise. So `callback` given to the `walk`
|
158 | * will fire only after all `iterator` callbacks willnotify they finished their
|
159 | * work:
|
160 | *
|
161 | * var iterator = function (path, stats, callback) {
|
162 | * // ... do something
|
163 | * if (err) {
|
164 | * // ... if error occured we can "stop" walker
|
165 | * callback(err);
|
166 | * return;
|
167 | * }
|
168 | * // ... if everything is good and finished notify walker we're done
|
169 | * callback();
|
170 | * };
|
171 | *
|
172 | * Iterator is called with following arguments:
|
173 | *
|
174 | * - `path` (String): Full path of the found element (e.g. `/foo/bar.txt`)
|
175 | * - `stats` (fs.Stats): Stats object of found path
|
176 | * - `callback` (Function): Callback function to call after path processing
|
177 | *
|
178 | *
|
179 | * ##### Example
|
180 | *
|
181 | * fstools.walk('/home/nodeca', function (path, stats, callback) {
|
182 | * if (stats.isBlockDevice()) {
|
183 | * callback(Error("WTF? Block devices are not expetcted in my room"));
|
184 | * return;
|
185 | * }
|
186 | *
|
187 | * if (stats.isSocket()) {
|
188 | * console.log("Finally I found my socket");
|
189 | * }
|
190 | *
|
191 | * callback();
|
192 | * }, function (err) {
|
193 | * if (err) {
|
194 | * // shit happens!
|
195 | * console.error(err);
|
196 | * process.exit(1);
|
197 | * }
|
198 | *
|
199 | * console.log("Hooray! We're done!");
|
200 | * });
|
201 | *
|
202 | *
|
203 | * ##### Example (using pattern matching)
|
204 | *
|
205 | * fstools.walk('/home/nodeca', '\.yml$', function (path, stats, callback) {
|
206 | * fs.readFile(path, 'utf-8', funtion (err, str) {
|
207 | * if (err) {
|
208 | * callback(err);
|
209 | * return;
|
210 | * }
|
211 | *
|
212 | * console.log(str);
|
213 | * callback();
|
214 | * });
|
215 | * }, function (err) {
|
216 | * if (err) {
|
217 | * console.error(err);
|
218 | * }
|
219 | *
|
220 | * console.log('Done!');
|
221 | * });
|
222 | **/
|
223 | fstools.walk = function (path, pattern, iterator, callback) {
|
224 | var match;
|
225 |
|
226 | if (!callback) {
|
227 | pattern = null;
|
228 | iterator = arguments[1];
|
229 | callback = arguments[2];
|
230 | }
|
231 |
|
232 | if (!pattern) {
|
233 | match = function () { return true; };
|
234 | } else if (_.isFunction(pattern) && !_.isRegExp(pattern)) {
|
235 | match = pattern;
|
236 | } else {
|
237 | pattern = new RegExp(pattern);
|
238 | match = function (path) { return pattern.test(path); };
|
239 | }
|
240 |
|
241 | path = path_normalize(path);
|
242 | fs.lstat(path, function (err, stats) {
|
243 | if (err) {
|
244 | callback('ENOENT' === err.code ? null : err);
|
245 | return;
|
246 | }
|
247 |
|
248 | if (stats.isDirectory()) {
|
249 | walk_deep(path, match, iterator, callback);
|
250 | return;
|
251 | }
|
252 |
|
253 | if (match(path)) {
|
254 | iterator(path, stats, callback);
|
255 | return;
|
256 | }
|
257 |
|
258 | callback();
|
259 | });
|
260 | };
|
261 |
|
262 |
|
263 | /**
|
264 | * FsTools.remove(path, callback) -> void
|
265 | * - path (String): Path to remove
|
266 | * - callback (Function): Fired after path was removed
|
267 | *
|
268 | * Removes given `path`. If it was a directory will remove it recursively,
|
269 | * similar to UNIX' `rm -rf <path>`. After all will fire `callback(err)` with
|
270 | * an error if there were any.
|
271 | *
|
272 | * If given `path` was file - will proxy call to `fs.unlink`.
|
273 | *
|
274 | *
|
275 | * ##### Example
|
276 | *
|
277 | * fstools.remove('/home/nodeca/trash', function (err) {
|
278 | * if (err) {
|
279 | * console.log("U can't touch that");
|
280 | * console.err(err);
|
281 | * process.exit(1);
|
282 | * } else {
|
283 | * console.log("It's Hammer time");
|
284 | * process.exit(0);
|
285 | * }
|
286 | * });
|
287 | **/
|
288 | fstools.remove = function (path, callback) {
|
289 | path = path_normalize(path);
|
290 | fs.lstat(path, function (err, stats) {
|
291 | if (err) {
|
292 | // file/dir not exists - no need to do anything
|
293 | if ('ENOENT' === err.code) {
|
294 | callback(null);
|
295 | return;
|
296 | }
|
297 |
|
298 | // unknown error - can't continue
|
299 | callback(err);
|
300 | return;
|
301 | }
|
302 |
|
303 | if (!stats.isDirectory()) {
|
304 | fs.unlink(path, callback);
|
305 | return;
|
306 | }
|
307 |
|
308 | async.series([
|
309 | async.apply(walk_flat, path, fstools.remove),
|
310 | async.apply(fs.rmdir, path)
|
311 | ], function (err, results) {
|
312 | callback(err);
|
313 | });
|
314 | });
|
315 | };
|
316 |
|
317 |
|
318 |
|
319 | /**
|
320 | * FsTools.removeSync(path) -> void
|
321 | * - path (String): Path to remove
|
322 | *
|
323 | * Removes given `path`. If it was a directory will remove it recursively,
|
324 | * similar to UNIX' `rm -rf <path>`.
|
325 | *
|
326 | * If given `path` was file - will proxy call to `fs.unlinkSync`.
|
327 | *
|
328 | *
|
329 | * ##### Example
|
330 | *
|
331 | * try {
|
332 | * fstools.remove('/home/nodeca/trash');
|
333 | * console.log("It's Hammer time");
|
334 | * process.exit(0);
|
335 | * } catch (err) {
|
336 | * console.log("U can't touch that");
|
337 | * console.err(err);
|
338 | * process.exit(1);
|
339 | * }
|
340 | **/
|
341 | fstools.removeSync = function removeSync(path) {
|
342 | var nested_err, lstat;
|
343 |
|
344 | path = path_normalize(path);
|
345 | lstat = fs.lstatSync(path);
|
346 |
|
347 | if (!lstat.isDirectory()) {
|
348 | fs.unlinkSync(path);
|
349 | return;
|
350 | }
|
351 |
|
352 | fs.readdirSync(path).forEach(function (file) {
|
353 | try {
|
354 | fstools.removeSync(path_join(path, file));
|
355 | } catch (err) {
|
356 | nested_err = err;
|
357 | }
|
358 | });
|
359 |
|
360 | if (!nested_err) {
|
361 | fs.rmdirSync(path);
|
362 | return;
|
363 | }
|
364 |
|
365 | throw nested_err;
|
366 | };
|
367 |
|
368 |
|
369 | /**
|
370 | * FsTools.mkdir(path, mode, callback) -> void
|
371 | * FsTools.mkdir(path, callback) -> void
|
372 | * - path (String): Path to create
|
373 | * - mode (String|Number): Permission mode of new directory. See stdlib
|
374 | * fs.mkdir for details. Default: '0755'.
|
375 | * - callback (Function): Fired after path was created
|
376 | *
|
377 | * Creates given path, creating parents recursively if needed.
|
378 | * Similar to UNIX' `mkdir -pf <path>`. After all will fire `callback(err)` with
|
379 | * an error if there were any.
|
380 | *
|
381 | *
|
382 | * ##### Example
|
383 | *
|
384 | * fstools.mkdir('/home/nodeca/media/xxx', function (err) {
|
385 | * if (err) {
|
386 | * console.log("Can't' create directory");
|
387 | * console.err(err);
|
388 | * process.exit(1);
|
389 | * } else {
|
390 | * console.log("We can now store some romantic movies here");
|
391 | * process.exit(0);
|
392 | * }
|
393 | * });
|
394 | **/
|
395 | fstools.mkdir = function (path, mode, callback) {
|
396 | if (undefined === callback && _.isFunction(mode)) {
|
397 | callback = mode;
|
398 | mode = '0755';
|
399 | }
|
400 |
|
401 | path = path_normalize(path);
|
402 | fs.exists(path, function (exists) {
|
403 | var parent;
|
404 |
|
405 | if (exists) {
|
406 | callback(null);
|
407 | return;
|
408 | }
|
409 |
|
410 | parent = dirname(path);
|
411 | fstools.mkdir(parent, mode, function (err) {
|
412 | if (err) {
|
413 | callback(err);
|
414 | return;
|
415 | }
|
416 |
|
417 | fs.mkdir(path, mode, function (err) {
|
418 | // EEXIST is not error in our case
|
419 | // but a race condition :((
|
420 | if (err && 'EEXIST' === err.code) {
|
421 | callback(null);
|
422 | return;
|
423 | }
|
424 |
|
425 | // fallback to default behavior
|
426 | callback(err);
|
427 | });
|
428 | });
|
429 | });
|
430 | };
|
431 |
|
432 |
|
433 | /**
|
434 | * FsTools.copy(src, dst, callback) -> void
|
435 | * - src (String): Source file
|
436 | * - dst (String): Destination file
|
437 | * - callback (Function): Fired after path has been copied
|
438 | *
|
439 | * Copies `src` to `dst`, creates directory for given `dst` with
|
440 | * [[FsTools.mkdir]] if needed. Fires `callback(err)` upon
|
441 | * completion.
|
442 | *
|
443 | *
|
444 | * ##### Example
|
445 | *
|
446 | * var src = '/home/nodeca/secrets.yml',
|
447 | * dst = '/home/nodeca/very/deep/secrets/main.yml';
|
448 | *
|
449 | * fstools.copy(src, dst, function (err) {
|
450 | * if (err) {
|
451 | * console.log("Failed copy " + src + " into " + dst);
|
452 | * console.err(err);
|
453 | * process.exit(1);
|
454 | * } else {
|
455 | * console.log("Done!");
|
456 | * process.exit(0);
|
457 | * }
|
458 | * });
|
459 | **/
|
460 | fstools.copy = function (src, dst, callback) {
|
461 | src = path_normalize(src);
|
462 | dst = path_normalize(dst);
|
463 |
|
464 | // sad but true - people make mistakes...
|
465 | if (src === dst) {
|
466 | callback(null);
|
467 | return;
|
468 | }
|
469 |
|
470 | fs.lstat(src, function (err, stats) {
|
471 | if (err) {
|
472 | callback(err);
|
473 | return;
|
474 | }
|
475 |
|
476 | fstools.mkdir(dirname(dst), function (err) {
|
477 | var chmod, done;
|
478 |
|
479 | if (err) {
|
480 | callback(err);
|
481 | return;
|
482 | }
|
483 |
|
484 | // chmod dst
|
485 | chmod = async.apply(fs.chmod, dst, stats.mode.toString(8).slice(-4));
|
486 |
|
487 | // reject async.series' results
|
488 | done = function (err/*, results */) { callback(err); };
|
489 |
|
490 | // *** file
|
491 | if (stats.isFile()) {
|
492 | async.series([async.apply(copy_file, src, dst), chmod], done);
|
493 | return;
|
494 | }
|
495 |
|
496 | // *** symbolic link
|
497 | if (stats.isSymbolicLink()) {
|
498 | async.waterfall([
|
499 | function (next) {
|
500 | fs.exists(dst, function (exists) {
|
501 | if (exists) {
|
502 | fstools.remove(dst, next);
|
503 | return;
|
504 | }
|
505 |
|
506 | next();
|
507 | });
|
508 | },
|
509 | async.apply(fs.readlink, src),
|
510 | function (linkpath, next) {
|
511 | fs.symlink(linkpath, dst, next);
|
512 | },
|
513 | chmod
|
514 | ], done);
|
515 | return;
|
516 | }
|
517 |
|
518 | // *** directory
|
519 | if (stats.isDirectory()) {
|
520 | async.series([
|
521 | function (next) {
|
522 | fs.mkdir(dst, '0755', function (err) {
|
523 | if (err && 'EEXIST' === err.code) {
|
524 | next(null);
|
525 | return;
|
526 | }
|
527 |
|
528 | next(err);
|
529 | });
|
530 | },
|
531 | async.apply(walk_flat, src, function (path, next) {
|
532 | fstools.copy(path, dst + path.replace(src, ''), next);
|
533 | }),
|
534 | chmod
|
535 | ], done);
|
536 | return;
|
537 | }
|
538 |
|
539 | // *** unsupported src
|
540 | callback(new Error("Unsupported type of the source"));
|
541 | });
|
542 | });
|
543 | };
|
544 |
|
545 |
|
546 | /**
|
547 | * FsTools.move(source, destination, callback) -> Void
|
548 | * - source (String): Source filename
|
549 | * - destination (String): Destination filename
|
550 | *
|
551 | * Moves file from `source` to `destination`.
|
552 | **/
|
553 | fstools.move = function move(source, destination, callback) {
|
554 | fs.lstat(source, function (err, stats) {
|
555 | if (err) {
|
556 | callback(err);
|
557 | return;
|
558 | }
|
559 |
|
560 | fs.rename(source, destination, function (err) {
|
561 | if (!err) {
|
562 | callback();
|
563 | return;
|
564 | }
|
565 |
|
566 | // TODO: Needs testing coverage
|
567 |
|
568 | // normally err.code can be:
|
569 | // - EXDEV (different partitions/devices)
|
570 | // - ENOTEMPTY (source and destination are not-empty dirs)
|
571 | // - EISDIR (destionation is dir, source is file)
|
572 |
|
573 | async.series([
|
574 | async.apply(fstools.copy, source, destination),
|
575 | async.apply(fstools.remove, source)
|
576 | ], function (err/*, results*/) {
|
577 | callback();
|
578 | });
|
579 | });
|
580 | });
|
581 | };
|
582 |
|
583 |
|
584 | /**
|
585 | * FsTools.tmpdir([template = '/tmp/fstools.XXXXXX']) -> String
|
586 | * - template (String): Temporary directory pattern.
|
587 | *
|
588 | * Returns non-existing (at the moment of request) temporary directory path.
|
589 | * `template` must contain a substring with at least 3 consecutive `X`, that
|
590 | * will be replaced with pseudo-random string of the same length.
|
591 | *
|
592 | *
|
593 | * ##### Example
|
594 | *
|
595 | * fstools.tmpdir('/tmp/fooXXX'); // -> '/tmp/fooa2f'
|
596 | * fstools.tmpdir('/tmp/fooXXX'); // -> '/tmp/foocb1'
|
597 | * fstools.tmpdir('/tmp/foo-XXXXXX'); // -> '/tmp/foo-ad25e0'
|
598 | **/
|
599 | fstools.tmpdir = function tmpdir(template) {
|
600 | var match = (template || '/tmp/fstools.XXXXXX').match(/^(.*?)(X{3,})(.*?)$/),
|
601 | attempts, length, random, pathname;
|
602 |
|
603 | if (!match) {
|
604 | throw new Error("Invalid tmpdir template: " + template);
|
605 | }
|
606 |
|
607 | attempts = 5;
|
608 | length = match[2].length;
|
609 |
|
610 | // Do not try more than attempts of times
|
611 | while (attempts--) {
|
612 | random = crypto.randomBytes(Math.ceil(length / 2)).toString('hex').substring(0, length);
|
613 | pathname = (match[1] || '') + random + (match[3] || '');
|
614 |
|
615 | if (!fs.existsSync(pathname)) {
|
616 | // Generated pathname is uniq - return it
|
617 | return pathname;
|
618 | }
|
619 | }
|
620 |
|
621 | throw new Error("Failed to generate uniq tmpdir with template: " + template);
|
622 | };
|