1 |
|
2 |
|
3 | 'use strict';
|
4 |
|
5 |
|
6 |
|
7 | var crypto = require('crypto');
|
8 | var fs = require('graceful-fs');
|
9 | var path = require('path');
|
10 | var url = require('url');
|
11 |
|
12 |
|
13 |
|
14 | var AppCache = require('@jokeyrhyme/appcache');
|
15 |
|
16 | var mkdirp = require('mkdirp');
|
17 | var request = require('request');
|
18 | var temp = require('temp').track();
|
19 |
|
20 |
|
21 |
|
22 | var FetcherIndex = require(path.join(__dirname, 'www', 'fetcher-index'));
|
23 |
|
24 | var doBrowserify = require(path.join(__dirname, 'lib', 'do-browserify'));
|
25 | var urlVars = require(path.join(__dirname, 'www', 'url_variations'));
|
26 |
|
27 |
|
28 |
|
29 |
|
30 |
|
31 |
|
32 |
|
33 | function Fetcher (opts) {
|
34 | this.date = new Date();
|
35 | this.remoteUrl = opts.remoteUrl;
|
36 | this.localPath = path.resolve(opts.localPath);
|
37 | this.tempPath = '';
|
38 | this.index = new FetcherIndex({ remoteUrl: this.remoteUrl });
|
39 | this.manifestUrl = '';
|
40 |
|
41 | this.transforms = {
|
42 | css: [],
|
43 | html: [],
|
44 | js: []
|
45 | };
|
46 | this.extractors = {
|
47 | manifestUrl: []
|
48 | };
|
49 |
|
50 | this.addExtractor('manifestUrl', require(path.join(__dirname, 'lib', 'extractors', 'manifestUrl.w3c')));
|
51 | this.addExtractor('requirejsSrc', require(path.join(__dirname, 'lib', 'extractors', 'manifestUrl.w3c')));
|
52 |
|
53 | this.addTransform('css', require(path.join(__dirname, 'lib', 'transforms', 'css.localUrls')));
|
54 | this.addTransform('html', require(path.join(__dirname, 'lib', 'transforms', 'html.removeManifest')));
|
55 | this.addTransform('html', require(path.join(__dirname, 'lib', 'transforms', 'html.localLinkHrefs')));
|
56 | this.addTransform('html', require(path.join(__dirname, 'lib', 'transforms', 'html.localScriptSrcs')));
|
57 | this.addTransform('html', require(path.join(__dirname, 'lib', 'transforms', 'html.injectAppCacheIndex')));
|
58 | this.addTransform('html', require(path.join(__dirname, 'lib', 'transforms', 'html.injectRequireJsShim')));
|
59 | }
|
60 |
|
61 | Fetcher.prototype.addExtractor = function (prop, fn) {
|
62 | if (!Array.isArray(this.extractors[prop])) {
|
63 | this.extractors[prop] = [];
|
64 | }
|
65 | this.extractors[prop].push(fn);
|
66 | };
|
67 |
|
68 | Fetcher.prototype.addTransform = function (ext, fn) {
|
69 | if (!Array.isArray(this.transforms[ext])) {
|
70 | this.transforms[ext] = [];
|
71 | }
|
72 | this.transforms[ext].push(fn);
|
73 | };
|
74 |
|
75 | Fetcher.prototype.afterTempPath = function () {
|
76 | var me = this;
|
77 |
|
78 | return new Promise(function (resolve, reject) {
|
79 | temp.mkdir('appcache-fetcher-' + me.date.valueOf(), function (err, dirPath) {
|
80 | if (err) {
|
81 | reject(err);
|
82 | return;
|
83 | }
|
84 | me.tempPath = dirPath;
|
85 | resolve(dirPath);
|
86 | });
|
87 | });
|
88 | };
|
89 |
|
90 | Fetcher.prototype.afterLocalPath = function () {
|
91 | var me = this;
|
92 |
|
93 | return new Promise(function (resolve, reject) {
|
94 | mkdirp(me.localPath, function (err) {
|
95 | if (err) {
|
96 | reject(err);
|
97 | return;
|
98 | }
|
99 | resolve(me.localPath);
|
100 | });
|
101 | });
|
102 | };
|
103 |
|
104 | Fetcher.prototype.readFile = function (filePath) {
|
105 | return new Promise(function (resolve, reject) {
|
106 | fs.readFile(filePath, { encoding: 'utf8' }, function (err, contents) {
|
107 | if (err) {
|
108 | console.error(err);
|
109 | reject(err);
|
110 | return;
|
111 | }
|
112 | resolve(contents);
|
113 | });
|
114 | });
|
115 | };
|
116 |
|
117 | Fetcher.prototype.writeFile = function (filePath, contents) {
|
118 | return new Promise(function (resolve, reject) {
|
119 | fs.writeFile(filePath, contents, function (err) {
|
120 | if (err) {
|
121 | console.error(err);
|
122 | reject(err);
|
123 | return;
|
124 | }
|
125 | resolve();
|
126 | });
|
127 | });
|
128 | };
|
129 |
|
130 | Fetcher.prototype.generateAppCacheIndexShim = function () {
|
131 | var addPath = path.join(__dirname, 'www', 'app-cache-index.js');
|
132 | var filePath = path.join(this.localPath, 'appCacheIndex.js');
|
133 | var options = {
|
134 | paths: [ this.localPath ],
|
135 | standalone: 'appCacheIndex'
|
136 | };
|
137 |
|
138 | return doBrowserify(addPath, filePath, options);
|
139 | };
|
140 |
|
141 | Fetcher.prototype.generateRequireShim = function () {
|
142 | var addPath = path.join(__dirname, 'www', 'require.load.js');
|
143 | var filePath = path.join(this.localPath, 'require.load.js');
|
144 |
|
145 | return doBrowserify(addPath, filePath, {});
|
146 | };
|
147 |
|
148 | Fetcher.prototype.download = function (remoteUrl, localPath) {
|
149 | var me = this;
|
150 | var filePath;
|
151 | var filename;
|
152 |
|
153 | if (Array.isArray(remoteUrl)) {
|
154 | return Promise.all(remoteUrl.map(function (r) {
|
155 | return me.download(r, localPath);
|
156 | }));
|
157 | }
|
158 |
|
159 | console.log('download:', remoteUrl);
|
160 |
|
161 | filename = me.generateLocalFilePath(remoteUrl);
|
162 | filePath = path.join(localPath, filename);
|
163 |
|
164 | return new Promise(function (resolve, reject) {
|
165 | var reader, writer;
|
166 |
|
167 | reader = request(remoteUrl)
|
168 | .on('error', function (err) {
|
169 | console.error(err);
|
170 | reject(err);
|
171 | })
|
172 | .on('response', function (res) {
|
173 | var errorMsg;
|
174 | if (res.statusCode !== 200) {
|
175 | errorMsg = remoteUrl + ' : ' + res.statusCode;
|
176 | console.error(errorMsg);
|
177 | reject(new Error(errorMsg));
|
178 | return;
|
179 | }
|
180 | me.index.set(remoteUrl, filename);
|
181 | });
|
182 |
|
183 | writer = fs.createWriteStream(filePath)
|
184 | .on('error', function (err) {
|
185 | console.error(err);
|
186 | reject(err);
|
187 | })
|
188 | .on('finish', function () {
|
189 | resolve();
|
190 | });
|
191 |
|
192 | reader.pipe(writer);
|
193 | });
|
194 | };
|
195 |
|
196 | Fetcher.prototype.generateLocalFilePath = function (remoteUrl) {
|
197 | var parsed;
|
198 | var hash;
|
199 | if (remoteUrl === this.remoteUrl) {
|
200 | return 'index.html';
|
201 | }
|
202 | if (remoteUrl === this.manifestUrl) {
|
203 | return 'appcache.manifest';
|
204 | }
|
205 | parsed = url.parse(remoteUrl, true, true);
|
206 | hash = crypto.createHash('sha1');
|
207 | hash.update(parsed.pathname + parsed.search);
|
208 | return hash.digest('hex') + path.extname(parsed.pathname);
|
209 | };
|
210 |
|
211 | Fetcher.prototype.persistFilesIndex = function () {
|
212 | var content = JSON.stringify(this.index, null, 2);
|
213 | var filePath = path.join(this.localPath, 'index.json');
|
214 |
|
215 | return this.writeFile(filePath, content);
|
216 | };
|
217 |
|
218 | Fetcher.prototype.getManifestURL = function () {
|
219 | var me = this;
|
220 | var filePath = path.join(this.localPath, 'index.html');
|
221 | var extractors = this.extractors.manifestUrl;
|
222 |
|
223 | if (!Array.isArray(extractors) || !extractors.length) {
|
224 | return Promise.reject(new Error('no manifestUrl extractors'));
|
225 | }
|
226 |
|
227 | return me.readFile(filePath)
|
228 | .then(function (contents) {
|
229 | var manifestUrl;
|
230 | var e, eLength, extractor;
|
231 | eLength = extractors.length;
|
232 | for (e = 0; e < eLength; e++) {
|
233 | extractor = extractors[e];
|
234 | manifestUrl = extractor({
|
235 | contents: contents,
|
236 | remoteUrl: me.remoteUrl
|
237 | });
|
238 | if (manifestUrl) {
|
239 | return Promise.resolve(manifestUrl);
|
240 | }
|
241 | }
|
242 | return Promise.resolve('');
|
243 | });
|
244 | };
|
245 |
|
246 | Fetcher.prototype.downloadAppCacheEntries = function () {
|
247 | var me = this;
|
248 | var appCache;
|
249 | var remoteUrls;
|
250 |
|
251 | delete require.cache[path.join(me.localPath, 'appcache.json')];
|
252 | appCache = require(path.join(me.localPath, 'appcache.json'));
|
253 |
|
254 | remoteUrls = appCache.cache.map(function (entry) {
|
255 | return url.resolve(me.remoteUrl, entry.replace(/^\/\//, 'https://'));
|
256 | });
|
257 |
|
258 | return this.download(remoteUrls, me.localPath);
|
259 | };
|
260 |
|
261 | Fetcher.prototype.postProcessFile = function (filePath) {
|
262 | var me = this;
|
263 | var ext = path.extname(filePath).toLowerCase().replace('.', '');
|
264 | var transforms = me.transforms[ext];
|
265 | if (!Array.isArray(transforms) || !transforms.length) {
|
266 | return Promise.resolve();
|
267 | }
|
268 | console.log('postProcessFile:', filePath.replace(process.cwd(), ''));
|
269 | return this.readFile(filePath)
|
270 | .then(function (contents) {
|
271 | var transformedContents = contents;
|
272 | var t, tLength, transform;
|
273 | tLength = transforms.length;
|
274 | for (t = 0; t < tLength; t++) {
|
275 | transform = transforms[t];
|
276 | transformedContents = transform({
|
277 | contents: transformedContents,
|
278 | filePath: filePath,
|
279 | index: me.index
|
280 | });
|
281 | }
|
282 | return Promise.resolve(transformedContents);
|
283 | })
|
284 | .then(function (contents) {
|
285 | return me.writeFile(filePath, contents);
|
286 | });
|
287 | };
|
288 |
|
289 | Fetcher.prototype.postProcessDownloads = function () {
|
290 | var me = this;
|
291 |
|
292 | return new Promise(function (resolve, reject) {
|
293 | fs.readdir(me.localPath, function (err, files) {
|
294 | if (err) {
|
295 | console.error(err);
|
296 | reject(err);
|
297 | return;
|
298 | }
|
299 | Promise.all(files.map(function (file) {
|
300 | return me.postProcessFile(path.join(me.localPath, file));
|
301 | })).then(resolve, reject);
|
302 | });
|
303 | });
|
304 | };
|
305 |
|
306 | Fetcher.prototype.go = function () {
|
307 | var me = this;
|
308 |
|
309 | return Promise.all([
|
310 | this.afterTempPath(),
|
311 | this.afterLocalPath()
|
312 | ])
|
313 | .then(function () {
|
314 | return me.download(me.remoteUrl, me.localPath);
|
315 | })
|
316 | .then(function () {
|
317 | return me.getManifestURL();
|
318 | })
|
319 | .then(function (manifestUrl) {
|
320 | me.manifestUrl = manifestUrl;
|
321 | return me.download(me.manifestUrl, me.localPath);
|
322 | })
|
323 | .then(function () {
|
324 | return me.readFile(path.join(me.localPath, 'appcache.manifest'));
|
325 | })
|
326 | .then(function (contents) {
|
327 | var appCache = AppCache.parse(contents);
|
328 | return me.writeFile(
|
329 | path.join(me.localPath, 'appcache.json'),
|
330 | JSON.stringify(appCache, null, 2)
|
331 | );
|
332 | })
|
333 | .then(function () {
|
334 | return me.downloadAppCacheEntries();
|
335 | })
|
336 | .then(function () {
|
337 | return me.persistFilesIndex();
|
338 | })
|
339 | .then(function () {
|
340 | return me.postProcessDownloads();
|
341 | })
|
342 | .then(function () {
|
343 | return Promise.all([
|
344 | me.generateAppCacheIndexShim(),
|
345 | me.generateRequireShim()
|
346 | ]);
|
347 | })
|
348 | .then(null, function (err) {
|
349 | console.error(err);
|
350 | return Promise.reject(err);
|
351 | });
|
352 | };
|
353 |
|
354 | Fetcher.getURLVariationsOnQuery = urlVars.getURLVariationsOnQuery;
|
355 | Fetcher.getURLVariationsOnScheme = urlVars.getURLVariationsOnScheme;
|
356 | Fetcher.getURLVariations = urlVars.getURLVariations;
|
357 |
|
358 | module.exports = Fetcher;
|