1 | 'use strict';
|
2 |
|
3 | const fs = require('fs-extra');
|
4 | const path = require('path');
|
5 | const crypto = require('crypto');
|
6 | const zlib = require('zlib');
|
7 | const EventEmitter = require('events').EventEmitter;
|
8 | const util = require('util');
|
9 |
|
10 | const _ = require('underscore');
|
11 | const async = require('async');
|
12 | const dir = require('node-dir');
|
13 | const uglifyjs = require('uglify-js');
|
14 | const CleanCss = require('clean-css');
|
15 | const less = require('less');
|
16 | const request = require('request');
|
17 |
|
18 | const S3 = require('./s3');
|
19 |
|
20 | const ONE_YEAR_IN_SECONDS = 31536000;
|
21 | const THIRTY_DAYS_IN_SECONDS = 2592000;
|
22 | const SEVEN_DAYS_IN_SECONDS = 604800;
|
23 |
|
24 |
|
25 |
|
26 |
|
27 |
|
28 | function AssetProcessor(config) {
|
29 |
|
30 | let baseRoot;
|
31 | let currRelativeRoot;
|
32 |
|
33 | this.config = config;
|
34 |
|
35 | if (config.s3) {
|
36 | this.s3 = new S3(config.s3.bucket, config.s3.key, config.s3.secret);
|
37 | if (config.s3.cloudfrontMapping) {
|
38 | this.s3.setCloudfrontMapping(config.s3.cloudfrontMapping);
|
39 | }
|
40 | }
|
41 |
|
42 |
|
43 |
|
44 | if (this.config.targets.javascripts && !this.config.targets.javascripts.extensions) {
|
45 | this.config.targets.javascripts.extensions = ['.js'];
|
46 | }
|
47 | if (this.config.targets.stylesheets && !this.config.targets.stylesheets.extensions) {
|
48 | this.config.targets.stylesheets.extensions = ['.css'];
|
49 | }
|
50 | if (this.config.targets.images && !this.config.targets.images.extensions) {
|
51 | this.config.targets.images.extensions = ['.ico', '.png','.jpg','.jpeg','.gif','.bmp','.svg'];
|
52 | }
|
53 |
|
54 |
|
55 |
|
56 | if (!this.config.root) {
|
57 | throw new Error('Configuration root is required');
|
58 | } else if (!fs.existsSync(this.config.root)) {
|
59 | throw new Error('Invalid configuration root');
|
60 | }
|
61 |
|
62 | this.config.root = path.normalize(this.config.root);
|
63 |
|
64 | baseRoot = this.config.root;
|
65 |
|
66 | currRelativeRoot = this.config.targets.javascripts && this.config.targets.javascripts.root;
|
67 | this.config.javascriptsRoot = path.normalize(currRelativeRoot ? path.resolve(baseRoot, currRelativeRoot) : baseRoot);
|
68 |
|
69 | currRelativeRoot = this.config.targets.stylesheets && this.config.targets.stylesheets.root;
|
70 | this.config.stylesheetsRoot = path.normalize(currRelativeRoot ? path.resolve(baseRoot, currRelativeRoot) : baseRoot);
|
71 |
|
72 | currRelativeRoot = this.config.targets.images && this.config.targets.images.root;
|
73 | this.config.imagesRoot = path.normalize(currRelativeRoot ? path.resolve(baseRoot, currRelativeRoot) : baseRoot);
|
74 | this.config.imagesRootRelative = path.relative(this.config.root, this.config.imagesRoot);
|
75 |
|
76 | currRelativeRoot = this.config.targets.extras && this.config.targets.extras.root;
|
77 | this.config.extrasRoot = path.normalize(currRelativeRoot ? path.resolve(baseRoot, currRelativeRoot) : baseRoot);
|
78 | this.config.extrasRootRelative = path.relative(this.config.root, this.config.extrasRoot);
|
79 |
|
80 |
|
81 | this.forceCdnUpdate = this.config.forceCdnUpdate;
|
82 | }
|
83 |
|
84 | util.inherits(AssetProcessor, EventEmitter);
|
85 |
|
86 |
|
87 |
|
88 |
|
89 |
|
90 | AssetProcessor.prototype.getJavaScriptFiles = function (normalizedFullPath, callback) {
|
91 | _getFilesRelativeToRoot(this.config.root, this.config.javascriptsRoot, this.config.targets.javascripts, normalizedFullPath, callback);
|
92 | };
|
93 |
|
94 |
|
95 |
|
96 |
|
97 |
|
98 | AssetProcessor.prototype.getCssFiles = function (normalizedFullPath, callback) {
|
99 | _getFilesRelativeToRoot(this.config.root, this.config.stylesheetsRoot, this.config.targets.stylesheets, normalizedFullPath, callback);
|
100 | };
|
101 |
|
102 |
|
103 |
|
104 |
|
105 |
|
106 | AssetProcessor.prototype.getImageFiles = function (normalizedFullPath, callback) {
|
107 | _getFilesRelativeToRoot(this.config.root, this.config.imagesRoot, this.config.targets.images, normalizedFullPath, callback);
|
108 | };
|
109 |
|
110 |
|
111 |
|
112 |
|
113 |
|
114 | AssetProcessor.prototype.getExtraFiles = function (normalizedFullPath, callback) {
|
115 | _getFilesRelativeToRoot(this.config.root, this.config.extrasRoot, this.config.targets.extras, normalizedFullPath, callback);
|
116 | };
|
117 |
|
118 | function _getFilesRelativeToRoot(mainRoot, activeRoot, targets, normalizedFullPath, callback) {
|
119 | const relative = path.relative(mainRoot, activeRoot);
|
120 |
|
121 | if (typeof normalizedFullPath === 'function') {
|
122 | callback = normalizedFullPath;
|
123 | normalizedFullPath = false;
|
124 | }
|
125 |
|
126 |
|
127 | _getFiles(targets, activeRoot, normalizedFullPath, function(err, files) {
|
128 | if (normalizedFullPath) {
|
129 |
|
130 | callback(err, files);
|
131 | } else {
|
132 |
|
133 |
|
134 | const relativeFiles = _.map(files || [], function (file) {
|
135 | return path.join(relative, file).replace(/\\/gi,'/');
|
136 | });
|
137 |
|
138 | callback(err, relativeFiles);
|
139 |
|
140 | }
|
141 | });
|
142 | }
|
143 |
|
144 |
|
145 |
|
146 |
|
147 |
|
148 | AssetProcessor.prototype.compileLessFiles = function(callback) {
|
149 | const self = this;
|
150 | const lessTargetConfig = JSON.parse(JSON.stringify(this.config.targets.stylesheets));
|
151 | const cssFiles = [];
|
152 |
|
153 | lessTargetConfig.extensions = ['.less'];
|
154 |
|
155 | async.auto({
|
156 | getFiles: [function(next) {
|
157 | _getFiles(lessTargetConfig, self.config.stylesheetsRoot, true, next);
|
158 | }],
|
159 | processFiles: ['getFiles', function(next, results) {
|
160 |
|
161 |
|
162 | const files = results.getFiles.filter(function(file) {
|
163 | return path.extname(file).toLowerCase() === '.less';
|
164 | });
|
165 |
|
166 | async.eachSeries(files, function(file, eachNext) {
|
167 |
|
168 | async.auto({
|
169 | readFile: [function(subNext) {
|
170 | fs.readFile(file, subNext);
|
171 | }],
|
172 | render: ['readFile', function(subNext, results) {
|
173 | less.render(results.readFile.toString(), {
|
174 | filename: path.resolve(file)
|
175 | }, subNext);
|
176 | }],
|
177 | writeFile: ['render', function(subNext, results) {
|
178 | const cssFile = file.replace(/\.less$/i, '.css');
|
179 | fs.writeFile(cssFile, results.render, function(err) {
|
180 | if (!err) {
|
181 | cssFiles.push(cssFile);
|
182 | }
|
183 | subNext(err);
|
184 | });
|
185 | }]
|
186 | }, function(err) {
|
187 | eachNext(err);
|
188 | });
|
189 | }, function(err) {
|
190 | next(err);
|
191 | });
|
192 | }]
|
193 | }, function(err) {
|
194 | callback(err, cssFiles);
|
195 | });
|
196 | };
|
197 |
|
198 |
|
199 |
|
200 |
|
201 |
|
202 |
|
203 | AssetProcessor.prototype.processJavaScript = function(excludeSourceMap, callback) {
|
204 | const self = this;
|
205 |
|
206 | if (typeof excludeSourceMap === 'function') {
|
207 | callback = excludeSourceMap;
|
208 | excludeSourceMap = false;
|
209 | }
|
210 |
|
211 | async.auto({
|
212 | getFiles: [function(next) {
|
213 | self.getJavaScriptFiles(true, next);
|
214 | }],
|
215 | getPath: ['getFiles', function(next, results) {
|
216 | _getJavaScriptPath(results.getFiles, next);
|
217 | }],
|
218 | minify: ['getPath', function(next, results) {
|
219 | const jsFiles = results.getFiles;
|
220 | let uglifyResult = {};
|
221 | let err;
|
222 | const mapPath = results.getPath.replace('.js','.map');
|
223 | const publicMapPath = self.s3 ? self.s3.urlWithBucket(mapPath) : mapPath;
|
224 | const uglifyOptions = excludeSourceMap ? {} : {outSourceMap: publicMapPath};
|
225 |
|
226 | self.emit('minifyStarted', {type: 'js', files: jsFiles});
|
227 |
|
228 | if (!jsFiles.length) {
|
229 | next(null, {});
|
230 | } else {
|
231 | try {
|
232 | uglifyResult = uglifyjs.minify(jsFiles, uglifyOptions);
|
233 | } catch (uglifyError) {
|
234 | err = uglifyError;
|
235 | }
|
236 |
|
237 | next(err, { js: uglifyResult && uglifyResult.code, map: uglifyResult.map });
|
238 | }
|
239 |
|
240 |
|
241 | }]
|
242 | }, callback);
|
243 | }
|
244 |
|
245 |
|
246 |
|
247 |
|
248 |
|
249 |
|
250 |
|
251 | AssetProcessor.prototype.saveJavaScriptToFile = function(callback, outputDir) {
|
252 |
|
253 | const self = this;
|
254 |
|
255 | async.auto({
|
256 | getSource: [function (next) {
|
257 | self.processJavaScript(false, next);
|
258 | }],
|
259 | saveFile: ['getSource', function (next, results) {
|
260 |
|
261 | let relativeUrl = "";
|
262 | const source = results.getSource.minify.js;
|
263 |
|
264 | if (source) {
|
265 | const basePath = outputDir ? outputDir : self.config.javascriptsRoot;
|
266 | const fullLocalPath = basePath + '/' + results.getSource.getPath;
|
267 | relativeUrl = '/' + results.getSource.getPath;
|
268 |
|
269 | fs.outputFileSync(fullLocalPath, source);
|
270 | }
|
271 |
|
272 | next(null, relativeUrl);
|
273 | }],
|
274 | }, function(err, results) {
|
275 | const result = {
|
276 | jsUrl: results && results.saveFile
|
277 | };
|
278 | callback(err, result);
|
279 | });
|
280 |
|
281 | };
|
282 |
|
283 |
|
284 |
|
285 |
|
286 |
|
287 |
|
288 | AssetProcessor.prototype.uploadJavaScriptToCdn = function(excludeSourceMap, callback) {
|
289 |
|
290 | const self = this;
|
291 |
|
292 | if (typeof excludeSourceMap === 'function') {
|
293 | callback = excludeSourceMap;
|
294 | excludeSourceMap = false;
|
295 | }
|
296 |
|
297 | async.auto({
|
298 | getSource: [function(next) {
|
299 | self.processJavaScript(excludeSourceMap, next);
|
300 | }],
|
301 | compressJs: ['getSource', function(next, results) {
|
302 | zlib.gzip(results.getSource.minify.js, next);
|
303 | }],
|
304 | uploadJs: ['compressJs', function(next, results) {
|
305 | const jsFiles = results.getSource.getFiles;
|
306 | const gzip = results.compressJs;
|
307 | const targetPath = results.getSource.getPath;
|
308 | const headers = {
|
309 | 'x-amz-acl': 'public-read',
|
310 | 'Content-Type': 'application/x-javascript',
|
311 | 'Content-Encoding': 'gzip',
|
312 | 'Content-Length': gzip.length,
|
313 | 'Cache-Control': 'public, max-age=' + ONE_YEAR_IN_SECONDS
|
314 | };
|
315 | self.emit('minifyEnded', {type: 'js', files: jsFiles});
|
316 | self.emit('uploadStarted', {type: 'js', target: targetPath, source: 'memory'});
|
317 | self.s3.putBuffer(gzip, targetPath, headers, next);
|
318 | }],
|
319 | uploadMap: ['getSource', function(next, results) {
|
320 | const mapPath = results.getSource.getPath.replace('.js','.map');
|
321 | if (!excludeSourceMap) {
|
322 | self.s3.putBuffer(results.getSource.minify.map, mapPath, next);
|
323 | } else {
|
324 | next();
|
325 | }
|
326 | }],
|
327 | uploadCompleted: ['uploadJs', function(next, results) {
|
328 | const targetPath = results.getSource.getPath;
|
329 | self.emit('uploadEnded', {type: 'js', target: targetPath, source: 'memory', url: results.uploadJs});
|
330 | next();
|
331 | }]
|
332 | }, function(err, results) {
|
333 | callback(err, results && results.uploadJs);
|
334 | });
|
335 | };
|
336 |
|
337 | AssetProcessor.prototype.processCss = function(callback) {
|
338 |
|
339 | const self = this;
|
340 |
|
341 | async.auto({
|
342 | getFiles: [function(next) {
|
343 | self.getCssFiles(true, next);
|
344 | }],
|
345 | getPath: ['getFiles', function(next, results) {
|
346 | _getCssPath(results.getFiles, next);
|
347 | }],
|
348 | cleanCss: ['getFiles', function(next, results) {
|
349 |
|
350 | const files = results.getFiles;
|
351 | let css = '';
|
352 |
|
353 | self.emit('minifyStarted', {type: 'css', files: files});
|
354 |
|
355 | async.eachSeries(files, function(file, eachNext) {
|
356 | let currCss = ''+fs.readFileSync(file);
|
357 | currCss = new CleanCss({noAdvanced: true, noRebase: true}).minify(currCss);
|
358 | currCss = _rebaseUrls(self, file, currCss);
|
359 | css += currCss+' \n';
|
360 | eachNext();
|
361 | }, function(err) {
|
362 | next(err, css);
|
363 | });
|
364 | }]
|
365 | }, callback);
|
366 | };
|
367 |
|
368 |
|
369 |
|
370 |
|
371 |
|
372 |
|
373 | AssetProcessor.prototype.saveCssToFile = function(callback, outputDir) {
|
374 |
|
375 | const self = this;
|
376 |
|
377 | async.auto({
|
378 | getSource: [function(next) {
|
379 | self.processCss(next)
|
380 | }],
|
381 | saveFile: ['getSource', function (next, results) {
|
382 |
|
383 | var relativeUrl = "";
|
384 | const source = results.getSource.cleanCss;
|
385 |
|
386 | if (source) {
|
387 | const basePath = outputDir ? outputDir : self.config.stylesheetsRoot;
|
388 | const fullLocalPath = basePath + '/' + results.getSource.getPath;
|
389 | relativeUrl = '/' + results.getSource.getPath;
|
390 |
|
391 | fs.outputFileSync(fullLocalPath, source);
|
392 | }
|
393 |
|
394 | next(null, relativeUrl);
|
395 | }],
|
396 | }, function(err, results) {
|
397 | callback(err, {
|
398 | cssUrl: results && results.saveFile
|
399 | });
|
400 | });
|
401 |
|
402 | };
|
403 |
|
404 |
|
405 |
|
406 |
|
407 |
|
408 | AssetProcessor.prototype.uploadCssToCdn = function(callback) {
|
409 |
|
410 | const self = this;
|
411 |
|
412 | async.auto({
|
413 | getSource: [function(next) {
|
414 | self.processCss(next)
|
415 | }],
|
416 | compressCss: ['getSource', function(next, results) {
|
417 | zlib.gzip(results.getSource.cleanCss, next);
|
418 | }],
|
419 | uploadCss: ['compressCss', function(next, results) {
|
420 | const cssFiles = results.getSource.getFiles;
|
421 | const gzip = results.compressCss;
|
422 | const targetPath = results.getSource.getPath;
|
423 | const headers = {
|
424 | 'x-amz-acl': 'public-read',
|
425 | 'Content-Type': 'text/css',
|
426 | 'Content-Encoding': 'gzip',
|
427 | 'Content-Length': gzip.length,
|
428 | 'Cache-Control': 'public, max-age=' + ONE_YEAR_IN_SECONDS
|
429 | };
|
430 |
|
431 | self.emit('minifyEnded', {type: 'css', files: cssFiles});
|
432 | self.emit('uploadStarted', {type: 'css', target: targetPath, source: 'memory'});
|
433 | self.s3.putBuffer(gzip, targetPath, headers, next);
|
434 | }],
|
435 | uploadCompleted: ['uploadCss', function(next, results) {
|
436 | const targetPath = results.getSource.getPath;
|
437 | self.emit('uploadEnded', {type: 'css', target: targetPath, source: 'memory', url: results.uploadCss});
|
438 | next();
|
439 | }]
|
440 | }, function(err, results) {
|
441 | callback(err, results && results.uploadCss);
|
442 | });
|
443 | };
|
444 |
|
445 | AssetProcessor.prototype.processImages = function(callback) {
|
446 |
|
447 | const self = this;
|
448 |
|
449 | async.auto({
|
450 | getFiles: [function(next) {
|
451 | self.getImageFiles(true, next);
|
452 | }],
|
453 | getFolder: [function(next) {
|
454 | _getImageFolder(self.config.imagesRootRelative, next);
|
455 | }],
|
456 | }, callback);
|
457 | };
|
458 |
|
459 |
|
460 |
|
461 |
|
462 |
|
463 | AssetProcessor.prototype.uploadImagesToCdn = function(callback) {
|
464 |
|
465 | const self = this;
|
466 |
|
467 | async.auto({
|
468 | getFiles: [function(next) {
|
469 | self.getImageFiles(true, next);
|
470 | }],
|
471 | getFolder: [function(next) {
|
472 | _getImageFolder(self.config.imagesRootRelative, next);
|
473 | }],
|
474 | uploadImages: ['getFiles', 'getFolder', function(next, results) {
|
475 | _wrapUploadEmitter(self, _uploadRelativeToRoot(results.getFiles, self.config.imagesRoot, results.getFolder, self.s3, next), 'image');
|
476 | }]
|
477 | }, function(err, results) {
|
478 | callback(err, results && results.uploadImages);
|
479 | });
|
480 | };
|
481 |
|
482 |
|
483 |
|
484 |
|
485 |
|
486 | AssetProcessor.prototype.uploadExtrasToCdn = function(callback) {
|
487 |
|
488 | const self = this;
|
489 |
|
490 | async.auto({
|
491 | getFiles: [function(next) {
|
492 | self.getExtraFiles(true, next);
|
493 | }],
|
494 | getFolder: [function(next) {
|
495 | _getExtrasFolder(self.config.extrasRootRelative, next);
|
496 | }],
|
497 | uploadExtras: ['getFiles', 'getFolder', function(next, results) {
|
498 | _wrapUploadEmitter(self, _uploadRelativeToRoot(results.getFiles, self.config.extrasRoot, results.getFolder, self.s3, next), 'extra');
|
499 | }]
|
500 | }, function(err, results) {
|
501 | callback(err, results && results.uploadExtras);
|
502 | });
|
503 | };
|
504 |
|
505 |
|
506 |
|
507 |
|
508 |
|
509 | AssetProcessor.prototype.importLatestStylesheets = function(callback) {
|
510 |
|
511 | const self = this;
|
512 |
|
513 | const gitToken = self.config.git && self.config.git.token;
|
514 | const imports = self.config.targets && self.config.targets.stylesheets && self.config.targets.stylesheets.imports || {};
|
515 |
|
516 | const destDir = path.resolve(self.config.stylesheetsRoot, imports.destination || '');
|
517 | const sources = imports.sources || [];
|
518 | const mappings = imports.mappings || {};
|
519 | let numImported = 0;
|
520 |
|
521 | if (!fs.existsSync(destDir)) {
|
522 | console.log('Creating import directory '+destDir);
|
523 | fs.mkdirsSync(destDir);
|
524 | }
|
525 |
|
526 | console.log('Importing '+sources.length+' css files to '+destDir);
|
527 | if (Object.keys(mappings).length) {
|
528 | console.log('Will be applying the following remappings:', mappings);
|
529 | }
|
530 |
|
531 | async.eachSeries(sources, function(source, eachNext) {
|
532 | const destPath = path.resolve(destDir, path.basename(source));
|
533 | console.log('Importing '+source+' to '+destPath);
|
534 | _importFromGit(source, destPath, gitToken, function(err) {
|
535 | if (!err) {
|
536 | _applyImportMappings(destPath, mappings, function(err) {
|
537 | if (!err) {
|
538 | numImported += 1;
|
539 | }
|
540 | eachNext(err);
|
541 | });
|
542 | } else {
|
543 | console.error(err);
|
544 | eachNext(err);
|
545 | }
|
546 | });
|
547 | }, function(err) {
|
548 | console.log('Imported '+numImported+' of '+sources.length+' stylesheets');
|
549 | callback(err, numImported);
|
550 | });
|
551 | };
|
552 |
|
553 |
|
554 |
|
555 |
|
556 |
|
557 |
|
558 |
|
559 | function _wrapUploadEmitter(parentEmitter, childEmitter, type) {
|
560 | childEmitter.on('uploadStarted', function(ev) {
|
561 | parentEmitter.emit('uploadStarted', {type: type, source: ev.source, target: ev.target});
|
562 | });
|
563 | childEmitter.on('uploadEnded', function(ev) {
|
564 | parentEmitter.emit('uploadEnded', {type: type, source: ev.source, target: ev.target, url: ev.url});
|
565 | });
|
566 | }
|
567 |
|
568 |
|
569 |
|
570 |
|
571 |
|
572 |
|
573 |
|
574 |
|
575 |
|
576 |
|
577 |
|
578 |
|
579 | AssetProcessor.prototype.ensureAssets = function(opts, callback) {
|
580 |
|
581 | if (typeof opts === 'function') {
|
582 | callback = opts;
|
583 | opts = {};
|
584 | }
|
585 |
|
586 | const self = this;
|
587 |
|
588 |
|
589 |
|
590 | async.auto({
|
591 | ensureJs: [function(next) {
|
592 | if (!opts.skipJs) {
|
593 | _checkAndUpdateJavaScript(self, next);
|
594 | } else {
|
595 | next();
|
596 | }
|
597 | }],
|
598 | ensureCss: ['ensureJs', function(next) {
|
599 | _checkAndUpdateCss(self, next);
|
600 | }],
|
601 | ensureImages: ['ensureCss', function(next) {
|
602 | _checkAndUpdateImages(self, next);
|
603 | }],
|
604 | ensureExtras: ['ensureImages', function(next) {
|
605 | _checkAndUpdateExtras(self, next);
|
606 | }]
|
607 | }, function(err, results) {
|
608 | const result = _.extend({}, results && results.ensureJs, results && results.ensureCss, results && results.ensureImages, results && results.ensureExtras);
|
609 | callback(err, result);
|
610 | });
|
611 |
|
612 | };
|
613 |
|
614 |
|
615 |
|
616 |
|
617 |
|
618 |
|
619 |
|
620 |
|
621 |
|
622 |
|
623 |
|
624 |
|
625 |
|
626 | AssetProcessor.prototype.processAssets = function(opts, callback) {
|
627 |
|
628 | if (typeof opts === 'function') {
|
629 | callback = opts;
|
630 | opts = {};
|
631 | }
|
632 |
|
633 | const self = this;
|
634 |
|
635 | async.auto({
|
636 | processJs: [function(next) {
|
637 | if (!opts.skipJs) {
|
638 | self.saveJavaScriptToFile(next, opts.outputDir);
|
639 | } else {
|
640 | next();
|
641 | }
|
642 | }],
|
643 | processCss: [function(next) {
|
644 | self.saveCssToFile(next, opts.outputDir);
|
645 | }]
|
646 | }, function(err, results) {
|
647 |
|
648 | let result = {};
|
649 |
|
650 | if (results) {
|
651 | result = _.extend({},
|
652 | results.processJs,
|
653 | results.processCss,
|
654 | {imagesUrl: ''},
|
655 | {extrasUrl: ''}
|
656 | );
|
657 | }
|
658 |
|
659 | callback(err, result);
|
660 | });
|
661 |
|
662 | };
|
663 |
|
664 |
|
665 |
|
666 |
|
667 |
|
668 |
|
669 |
|
670 |
|
671 |
|
672 |
|
673 |
|
674 |
|
675 | function _uploadRelativeToRoot(files, activeRoot, targetFolder, s3, callback) {
|
676 | const eventEmitter = new EventEmitter();
|
677 | async.eachSeries(files, function(file, eachNext) {
|
678 | const target = targetFolder+'/'+_stripRoot(file, activeRoot);
|
679 | const headers = {
|
680 | 'x-amz-acl': 'public-read',
|
681 | 'Cache-Control': 'public, max-age=' + SEVEN_DAYS_IN_SECONDS
|
682 | };
|
683 | eventEmitter.emit('uploadStarted', {target: target, source: file});
|
684 | s3.putFile(file, target, headers, function(err, result) {
|
685 | eventEmitter.emit('uploadEnded', {source: file, target: target, url: result});
|
686 | eachNext(err);
|
687 | });
|
688 | }, function(err) {
|
689 | callback(err, s3.urlWithBucket(targetFolder));
|
690 | });
|
691 | return eventEmitter;
|
692 | }
|
693 |
|
694 |
|
695 |
|
696 |
|
697 |
|
698 |
|
699 |
|
700 |
|
701 |
|
702 |
|
703 | function _stripRoot(file, root) {
|
704 | const posixRoot = root && root.replace(/\\/g, '/');
|
705 | file = file.replace(/\\/g, '/');
|
706 | const index = file.indexOf(posixRoot);
|
707 | if (index !== -1) {
|
708 | file = file.substr(posixRoot.length);
|
709 | if (file[0] === '/') {
|
710 |
|
711 | file = file.substr(1);
|
712 | }
|
713 | }
|
714 | return file;
|
715 | }
|
716 |
|
717 |
|
718 |
|
719 |
|
720 |
|
721 |
|
722 |
|
723 |
|
724 |
|
725 | function _getFiles(config, activeRoot, normalizedFullPath, callback) {
|
726 |
|
727 | if (typeof normalizedFullPath === 'function') {
|
728 | callback = normalizedFullPath;
|
729 | normalizedFullPath = false;
|
730 | }
|
731 |
|
732 | async.auto({
|
733 | getRelevantFiles: [function(next) {
|
734 |
|
735 | let directories = config.directories;
|
736 | let specificFiles = config.files;
|
737 |
|
738 | if (!directories && !specificFiles) {
|
739 | directories = activeRoot;
|
740 | }
|
741 | if (directories && !(directories instanceof Array)) {
|
742 | directories = [directories];
|
743 | }
|
744 |
|
745 |
|
746 | directories = _.map(directories || [], function(directory) {
|
747 | return path.resolve(activeRoot, directory);
|
748 | });
|
749 |
|
750 |
|
751 | specificFiles = _.map(specificFiles || [], function(specificFile) {
|
752 | return path.resolve(activeRoot, specificFile);
|
753 | });
|
754 |
|
755 | const extensions = config.extensions || [];
|
756 | const preference = config.preference || [];
|
757 | const exclusions = config.exclude || [];
|
758 |
|
759 | _getRelevantFiles(activeRoot, directories, specificFiles, extensions, preference, exclusions, next);
|
760 | }],
|
761 | normalizeFullPathIfNeeded: ['getRelevantFiles', function(next, results) {
|
762 | let files = results.getRelevantFiles;
|
763 | if (normalizedFullPath) {
|
764 | files = _.map(files, function(file) {
|
765 | if (file[0] !== '/') {
|
766 | return path.normalize(path.join(activeRoot, file));
|
767 | }
|
768 | });
|
769 | }
|
770 | next(null, files);
|
771 | }]
|
772 | }, function(err, results) {
|
773 | callback(err, results && results.normalizeFullPathIfNeeded);
|
774 | });
|
775 | }
|
776 |
|
777 |
|
778 |
|
779 |
|
780 |
|
781 |
|
782 |
|
783 |
|
784 |
|
785 |
|
786 |
|
787 |
|
788 | function _getRelevantFiles(root, directories, specificFiles, extensions, preference, exclusions, callback) {
|
789 |
|
790 | async.auto({
|
791 | files: [function(next) {
|
792 |
|
793 | let files = specificFiles || [];
|
794 | async.each(directories, function(directory, eachNext) {
|
795 | dir.files(directory, function(err, dirFiles) {
|
796 | files = files.concat(dirFiles || []);
|
797 | eachNext(err);
|
798 | });
|
799 | }, function(err) {
|
800 | next(err, files);
|
801 | });
|
802 | }],
|
803 | process: ['files', function(next, results) {
|
804 |
|
805 |
|
806 | let files = _.map(results.files, function(file) {
|
807 | return _stripRoot(file, root);
|
808 | });
|
809 |
|
810 |
|
811 | files = files.filter(function(file) {
|
812 |
|
813 | const extensionMatch = _extensionMatch(file, extensions);
|
814 |
|
815 |
|
816 | const excluded = (exclusions || []).some(function(excludePath) {
|
817 | return file.toLowerCase().indexOf(excludePath.toLowerCase()) === 0;
|
818 | });
|
819 |
|
820 |
|
821 | const included = (preference || []).some(function(preference) {
|
822 | return file.toLowerCase() === preference.toLowerCase();
|
823 | });
|
824 |
|
825 | let include = false;
|
826 | if (included) {
|
827 | include = true;
|
828 | } else if (extensionMatch) {
|
829 | include = !excluded;
|
830 | }
|
831 |
|
832 |
|
833 | return include;
|
834 | });
|
835 |
|
836 |
|
837 | preference = (preference || []).concat(_.map(specificFiles, function(specificFile) {
|
838 | return _stripRoot(specificFile, root);
|
839 | }));
|
840 |
|
841 |
|
842 | files.sort(function(a,b) {
|
843 | const preferenceDiff = _preference(preference, a) - _preference(preference, b);
|
844 | return preferenceDiff !== 0 ? preferenceDiff : a < b ? -1 : a > b ? 1 : 0;
|
845 | });
|
846 |
|
847 |
|
848 | preference.forEach(function(preferencePath) {
|
849 | const fullPath = path.resolve(root, preferencePath);
|
850 |
|
851 | if (!fs.existsSync(fullPath) && (!path.extname(fullPath) || _extensionMatch(fullPath, extensions))) {
|
852 | console.warn('Warning: '+preferencePath+' was in configuration but cannot be found');
|
853 | }
|
854 | });
|
855 |
|
856 | next(null, files);
|
857 | }]
|
858 | }, function(err, results) {
|
859 | callback(err, results && results.process);
|
860 | });
|
861 | }
|
862 |
|
863 |
|
864 |
|
865 |
|
866 |
|
867 |
|
868 |
|
869 |
|
870 | function _preference(preferences, fileName) {
|
871 |
|
872 | let i;
|
873 | let currPreference;
|
874 |
|
875 | fileName = fileName.toLowerCase();
|
876 |
|
877 | for (i = 0; i < preferences.length; i += 1) {
|
878 | currPreference = preferences[i].toLowerCase();
|
879 | if (path.extname(currPreference)) {
|
880 |
|
881 | if (currPreference === fileName) {
|
882 | break;
|
883 | }
|
884 | } else {
|
885 |
|
886 |
|
887 | if (path.relative(currPreference, fileName).indexOf('..') === -1) {
|
888 | break;
|
889 | }
|
890 | }
|
891 | }
|
892 |
|
893 | return i;
|
894 | }
|
895 |
|
896 |
|
897 |
|
898 |
|
899 |
|
900 |
|
901 |
|
902 | function _hashFiles(files, callback) {
|
903 |
|
904 | const hash = crypto.createHash('md5');
|
905 |
|
906 | async.eachSeries(files, function(file, eachNext) {
|
907 | fs.readFile(file, function(err, data) {
|
908 | hash.update(data);
|
909 | eachNext(err);
|
910 | });
|
911 | }, function(err) {
|
912 | callback(err, !err && files.length && hash.digest('hex'));
|
913 | });
|
914 | }
|
915 |
|
916 | function _getJavaScriptPath(files, callback) {
|
917 | _hashAndGeneratePath(files, 'js', '.js', callback);
|
918 | }
|
919 |
|
920 | function _getCssPath(files, callback) {
|
921 | _hashAndGeneratePath(files, 'css', '.css', callback);
|
922 | }
|
923 |
|
924 | function _getImageFolder(imagesRootRelative, callback) {
|
925 | return _firstDirNameFromRootPath(imagesRootRelative, 'img', callback);
|
926 | }
|
927 |
|
928 | function _getExtrasFolder(extrasRootRelative, callback) {
|
929 | return _firstDirNameFromRootPath(extrasRootRelative, 'extra', callback);
|
930 | }
|
931 |
|
932 | function _firstDirNameFromRootPath(rootPath, defaultDirName, callback) {
|
933 | const folder = path.sep+(_firstDirName(rootPath) || defaultDirName);
|
934 | if (callback) {
|
935 | callback(null, folder);
|
936 | }
|
937 | return folder;
|
938 | }
|
939 |
|
940 | function _firstDirName(dirPath) {
|
941 | dirPath = (dirPath || '').replace(/[\\\/]/g, path.sep);
|
942 |
|
943 | if (dirPath[0] === path.sep) {
|
944 | dirPath = dirPath.substr(1);
|
945 | }
|
946 |
|
947 | return dirPath.split(path.sep)[0] || '';
|
948 | }
|
949 |
|
950 | function _hashAndGeneratePath(files, path, extension, callback) {
|
951 | async.auto({
|
952 | hashFiles: [function(next) {
|
953 | _hashFiles(files, next);
|
954 | }],
|
955 | path: ['hashFiles', function(next, results) {
|
956 | const hash = results.hashFiles;
|
957 | const uri = path+'/'+hash+extension;
|
958 | next(null, uri);
|
959 | }]
|
960 | }, function(err, results) {
|
961 | callback(err, results && results.path);
|
962 | });
|
963 | }
|
964 |
|
965 | function _checkAndUpdateJavaScript(self, callback) {
|
966 |
|
967 | async.auto({
|
968 | getJsFiles: [function (next) {
|
969 | self.getJavaScriptFiles(true, next);
|
970 | }],
|
971 | getJsPath: ['getJsFiles', function (next, results) {
|
972 | _getJavaScriptPath(results.getJsFiles, next);
|
973 | }],
|
974 | checkJs: ['getJsPath', function (next, results) {
|
975 | self.s3.fileExists(results.getJsPath, next);
|
976 | }],
|
977 | updateJsIfNeeded: ['checkJs', function (next, results) {
|
978 | const jsChanged = !results.checkJs;
|
979 |
|
980 | self.emit('filesChecked', {type: 'js', changed: jsChanged});
|
981 |
|
982 | if (jsChanged || self.forceCdnUpdate) {
|
983 |
|
984 | self.uploadJavaScriptToCdn(function (err, newUrl) {
|
985 | next(err, newUrl);
|
986 | });
|
987 | } else {
|
988 |
|
989 | next(null, self.s3.urlWithBucket(results.getJsPath));
|
990 | }
|
991 | }]
|
992 | }, function(err, results) {
|
993 | const result = {
|
994 | jsUrl: results && results.updateJsIfNeeded,
|
995 | jsChanged: results && !results.checkJs
|
996 | };
|
997 | callback(err, result);
|
998 | });
|
999 | }
|
1000 |
|
1001 | function _checkAndUpdateCss(self, callback) {
|
1002 |
|
1003 | async.auto({
|
1004 | getCssFiles: [function (next) {
|
1005 | self.getCssFiles(true, next);
|
1006 | }],
|
1007 | getCssPath: ['getCssFiles', function (next, results) {
|
1008 | _getCssPath(results.getCssFiles, next);
|
1009 | }],
|
1010 | checkCss: ['getCssPath', function (next, results) {
|
1011 | self.s3.fileExists(results.getCssPath, next);
|
1012 | }],
|
1013 | updateCssIfNeeded: ['checkCss', function (next, results) {
|
1014 | const cssChanged = !results.checkCss;
|
1015 |
|
1016 | self.emit('filesChecked', {type: 'css', changed: cssChanged});
|
1017 |
|
1018 | if (cssChanged || self.forceCdnUpdate) {
|
1019 |
|
1020 | self.uploadCssToCdn(function (err, newUrl) {
|
1021 | next(err, newUrl);
|
1022 | });
|
1023 | } else {
|
1024 |
|
1025 | next(null, self.s3.urlWithBucket(results.getCssPath));
|
1026 | }
|
1027 | }]
|
1028 | }, function(err, results) {
|
1029 | const result = {
|
1030 | cssUrl: results && results.updateCssIfNeeded,
|
1031 | cssChanged: results && !results.checkCss
|
1032 | };
|
1033 | callback(err, result);
|
1034 | });
|
1035 | }
|
1036 |
|
1037 | function _checkAndUpdateImages(self, callback) {
|
1038 | _checkRelativeToRoot(self, 'image', callback);
|
1039 | }
|
1040 |
|
1041 | function _checkAndUpdateExtras(self, callback) {
|
1042 | _checkRelativeToRoot(self, 'extra', callback);
|
1043 | }
|
1044 |
|
1045 | function _checkRelativeToRoot(self, type, callback) {
|
1046 |
|
1047 | async.auto({
|
1048 | getFiles: [function(next) {
|
1049 | if (type === 'image') {
|
1050 | self.getImageFiles(true, next);
|
1051 | } else {
|
1052 | self.getExtraFiles(true, next);
|
1053 | }
|
1054 | }],
|
1055 | getFolder: [function(next) {
|
1056 | if (type === 'image') {
|
1057 | _getImageFolder(self.config.imagesRootRelative, next);
|
1058 | } else {
|
1059 | _getExtrasFolder(self.config.extrasRootRelative, next);
|
1060 | }
|
1061 | }],
|
1062 | hashAndGeneratePath: ['getFiles', 'getFolder', function(next, results) {
|
1063 | _hashAndGeneratePath(results.getFiles, results.getFolder, '.txt', next);
|
1064 | }],
|
1065 | checkFiles: ['hashAndGeneratePath', function(next, results) {
|
1066 | self.s3.fileExists(results.hashAndGeneratePath, next);
|
1067 | }],
|
1068 | updateIfNeeded: ['checkFiles', function(next, results) {
|
1069 | const filesChanged = !results.checkFiles;
|
1070 | self.emit('filesChecked', {type: type, changed: filesChanged});
|
1071 |
|
1072 | if (filesChanged || self.forceCdnUpdate) {
|
1073 | if (type === 'image') {
|
1074 | self.uploadImagesToCdn(next);
|
1075 | } else {
|
1076 | self.uploadExtrasToCdn(next);
|
1077 | }
|
1078 | } else {
|
1079 | next(null, self.s3.urlWithBucket(results.getFolder));
|
1080 | }
|
1081 | }],
|
1082 | uploadIndicatorFileIfNeeded: ['updateIfNeeded', function(next, results) {
|
1083 | const targetPath = results.hashAndGeneratePath;
|
1084 | if (!results.checkFiles) {
|
1085 | self.emit('uploadStarted', {type: type, target: targetPath, source: 'memory'});
|
1086 | self.s3.putBuffer(results.getFiles.join('\n'), targetPath, function(err, result) {
|
1087 | self.emit('uploadEnded', {type: type, target: targetPath, source: 'memory', url: result});
|
1088 | next(err);
|
1089 | });
|
1090 | } else {
|
1091 | next();
|
1092 | }
|
1093 | }]
|
1094 | }, function(err, results) {
|
1095 | const result = {};
|
1096 |
|
1097 |
|
1098 | type = type+'s';
|
1099 |
|
1100 | result[type+'Url'] = results && results.updateIfNeeded;
|
1101 | result[type+'Changed'] = !results.checkFiles;
|
1102 |
|
1103 | callback(err, result);
|
1104 | });
|
1105 |
|
1106 | }
|
1107 |
|
1108 |
|
1109 |
|
1110 |
|
1111 |
|
1112 |
|
1113 |
|
1114 |
|
1115 |
|
1116 |
|
1117 | function _rebaseUrls(self, file, css) {
|
1118 |
|
1119 | const imgRoot = self.config.imagesRoot;
|
1120 | const imgRebaseRoot = _getImageFolder(self.config.imagesRootRelative);
|
1121 | const extrasRoot = self.config.extrasRoot;
|
1122 | const extrasRebaseRoot = _getExtrasFolder(self.config.extrasRootRelative);
|
1123 | let activeSourceRoot;
|
1124 | let activeRebaseRoot;
|
1125 |
|
1126 | let normalizedUrl;
|
1127 | let assetPath;
|
1128 | let rebasedUrl;
|
1129 |
|
1130 |
|
1131 | const urlRegex = /url\((['"]?([^'"\)]+)['"]?)\)/ig;
|
1132 | const externalHttpRegex = /^(http(s)?:)?\/\//i;
|
1133 | const absoluteUrlRegex = /^[\/\\][^\/]/;
|
1134 | const base64Regex = /^data:.+;base64/i;
|
1135 |
|
1136 | let match;
|
1137 | let url;
|
1138 | let startIndex;
|
1139 | let endIndex;
|
1140 |
|
1141 | file = path.normalize(file);
|
1142 |
|
1143 | while ((match = urlRegex.exec(css)) !== null) {
|
1144 | url = match[1];
|
1145 | startIndex = match.index+match[0].indexOf(url);
|
1146 | endIndex = startIndex + url.length;
|
1147 | activeSourceRoot = null;
|
1148 |
|
1149 |
|
1150 | url = url.replace(/^['"]/, '').replace(/['"]$/, '');
|
1151 |
|
1152 |
|
1153 | if (!externalHttpRegex.test(url) && !base64Regex.test(url)) {
|
1154 |
|
1155 | normalizedUrl = path.normalize(url);
|
1156 | if (_extensionMatch(normalizedUrl, self.config.targets.images && self.config.targets.images.extensions)) {
|
1157 | activeSourceRoot = imgRoot;
|
1158 | activeRebaseRoot = imgRebaseRoot;
|
1159 | } else {
|
1160 | activeSourceRoot = extrasRoot;
|
1161 | activeRebaseRoot = extrasRebaseRoot;
|
1162 | }
|
1163 | }
|
1164 |
|
1165 |
|
1166 | if (activeSourceRoot) {
|
1167 |
|
1168 | assetPath = absoluteUrlRegex.test(normalizedUrl) ? path.join(activeSourceRoot, normalizedUrl) : path.resolve(path.dirname(file), normalizedUrl);
|
1169 | if (assetPath.indexOf(activeSourceRoot) === 0) {
|
1170 |
|
1171 | if (self.s3) {
|
1172 | rebasedUrl = self.s3.urlWithBucket(path.join(activeRebaseRoot, assetPath.substr(activeSourceRoot.length)));
|
1173 | } else {
|
1174 | rebasedUrl = path.join(activeRebaseRoot, assetPath.substr(activeSourceRoot.length));
|
1175 | }
|
1176 |
|
1177 |
|
1178 | css = css.substr(0, startIndex)+rebasedUrl+css.substr(endIndex);
|
1179 |
|
1180 | urlRegex.lastIndex = endIndex;
|
1181 | } else {
|
1182 | console.warn('Cannot find expected root '+activeSourceRoot+' in asset path '+assetPath);
|
1183 | }
|
1184 | }
|
1185 | }
|
1186 |
|
1187 | return css;
|
1188 | }
|
1189 |
|
1190 |
|
1191 |
|
1192 |
|
1193 |
|
1194 |
|
1195 |
|
1196 |
|
1197 |
|
1198 | function _importFromGit(sourceUrl, destPath, accessToken, callback) {
|
1199 |
|
1200 | const apiUrl = _toGitHubApiSource(sourceUrl);
|
1201 | const requestOptions = {
|
1202 | url: apiUrl,
|
1203 | headers: {
|
1204 | Authorization: 'token '+accessToken,
|
1205 | Accept: 'application/vnd.github.v3.raw',
|
1206 | 'User-Agent': 'NodeBot (+http://videoblocks.com)'
|
1207 | }
|
1208 | };
|
1209 |
|
1210 | const ws = fs.createWriteStream(destPath);
|
1211 |
|
1212 |
|
1213 | const req = request(requestOptions);
|
1214 | let statusCode;
|
1215 | let reqErr;
|
1216 |
|
1217 | req.on('response', function(response) {
|
1218 | statusCode = response.statusCode;
|
1219 | });
|
1220 |
|
1221 | req.on('end', function() {
|
1222 | const success = statusCode === 200;
|
1223 | reqErr = reqErr || (!success ? new Error('Error requesting '+apiUrl+'; status code '+statusCode) : null);
|
1224 | });
|
1225 |
|
1226 | req.on('error', function(err) {
|
1227 | reqErr = reqErr || err;
|
1228 | });
|
1229 |
|
1230 | ws.on('close', function() {
|
1231 | const success = statusCode === 200;
|
1232 | callback(reqErr, success);
|
1233 | });
|
1234 |
|
1235 | req.pipe(ws);
|
1236 | }
|
1237 |
|
1238 | function _applyImportMappings(filePath, mappings, callback) {
|
1239 |
|
1240 | async.auto({
|
1241 | readFile: [function(next) {
|
1242 | fs.readFile(filePath, 'utf8', next);
|
1243 | }],
|
1244 | replaceMappings: ['readFile', function(next, results) {
|
1245 | let css = results.readFile;
|
1246 |
|
1247 | Object.keys(mappings || {}).forEach(function(importedRoot) {
|
1248 | const search = _escapeRegExp(importedRoot);
|
1249 | const replacement = mappings[importedRoot];
|
1250 | css = css.replace(new RegExp(search, 'g'), replacement);
|
1251 | });
|
1252 |
|
1253 | next(null, css);
|
1254 | }],
|
1255 | writeFile: ['replaceMappings', function(next, results) {
|
1256 | const remappedCss = results.replaceMappings;
|
1257 | fs.writeFile(filePath, remappedCss, next);
|
1258 | }]
|
1259 | }, function(err) {
|
1260 | callback(err);
|
1261 | });
|
1262 | }
|
1263 |
|
1264 |
|
1265 |
|
1266 |
|
1267 |
|
1268 |
|
1269 |
|
1270 |
|
1271 |
|
1272 |
|
1273 | function _toGitHubApiSource(sourceUrl) {
|
1274 |
|
1275 |
|
1276 | const gitHubBlobUrlRegex = /^https?:\/\/github\.com\/([^\/]+)\/([^\/]+)\/blob\/[^\/]+\/(.*)$/;
|
1277 | const match = gitHubBlobUrlRegex.exec(sourceUrl);
|
1278 | let org;
|
1279 | let repo;
|
1280 | let path;
|
1281 | if (match) {
|
1282 |
|
1283 | org = match[1];
|
1284 | repo = match[2];
|
1285 | path = match[3];
|
1286 | sourceUrl = 'https://api.github.com/repos/'+org+'/'+repo+'/contents/'+path;
|
1287 | }
|
1288 |
|
1289 | if (!sourceUrl.match(/^https?:\/\/api\.github\.com\/repos\//)) {
|
1290 | throw new Error('Invalid github import source URL: '+sourceUrl);
|
1291 | }
|
1292 |
|
1293 | return sourceUrl;
|
1294 | }
|
1295 |
|
1296 |
|
1297 |
|
1298 |
|
1299 |
|
1300 |
|
1301 |
|
1302 |
|
1303 |
|
1304 | function _extensionMatch(filePath, extensions) {
|
1305 | filePath = (filePath || '').toLowerCase();
|
1306 | return !extensions || !extensions.length || extensions.some(function(extension) {
|
1307 | return path.extname(filePath).toLowerCase() === extension.toLowerCase();
|
1308 | });
|
1309 | }
|
1310 |
|
1311 | function _escapeRegExp(string) {
|
1312 | return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
|
1313 | }
|
1314 |
|
1315 | module.exports = AssetProcessor;
|