1 | var crypto = require ('crypto'),
|
2 | util = require ('util'),
|
3 | urlUtil = require ('url'),
|
4 | path = require ('path'),
|
5 | os = require ('os'),
|
6 | task = require ('./base'),
|
7 | urlModel = require ('../model/from-url'),
|
8 | io = require ('../io/easy');
|
9 |
|
10 |
|
11 | var cacheTask = module.exports = function (config) {
|
12 |
|
13 | try {
|
14 | if (typeof project === "undefined") {
|
15 |
|
16 |
|
17 |
|
18 | this.cachePath = new io (os.tmpdir())
|
19 | } else {
|
20 | if (!project.config.cachePath) {
|
21 | console.log ('cachePath not defined in project config!');
|
22 | }
|
23 | this.cachePath = project.root.file_io (project.config.cachePath);
|
24 | }
|
25 |
|
26 | if (!cacheTask.caching) {
|
27 | cacheTask.caching = {};
|
28 | }
|
29 | } catch (e) {
|
30 | console.log (e);
|
31 | }
|
32 |
|
33 |
|
34 | this.url = config.url;
|
35 |
|
36 | this.init (config);
|
37 |
|
38 | if (!this.timeout)
|
39 | this.timeout = 10000;
|
40 |
|
41 | };
|
42 |
|
43 | util.inherits (cacheTask, task);
|
44 |
|
45 | util.extend (cacheTask.prototype, {
|
46 | |
47 |
|
48 |
|
49 |
|
50 | generateCacheFileName: function () {
|
51 |
|
52 | if (this.cacheFilePath)
|
53 | return this.cacheFilePath;
|
54 |
|
55 | var shasum = crypto.createHash('sha1');
|
56 | shasum.update(this.url.href);
|
57 | this.cacheFile = this.cachePath.file_io (shasum.digest('hex'));
|
58 | this.cacheFilePath = this.cacheFile.path;
|
59 | this.cacheFileName = path.basename(this.cacheFile.path);
|
60 |
|
61 | return this.cacheFilePath;
|
62 | }
|
63 | });
|
64 |
|
65 | cacheTask.prototype.initModel = function () {
|
66 | var self = this;
|
67 |
|
68 | if (Object.is('String', this.url)) {
|
69 | try {
|
70 | this.url = urlUtil.parse(this.url, true);
|
71 | } catch (e) {
|
72 | this.emitError(e);
|
73 | return;
|
74 | }
|
75 | }
|
76 |
|
77 | if (!this.url.href) {
|
78 | this.url.href = urlUtil.format (this.url);
|
79 | }
|
80 |
|
81 | this.url.headers = this.headers || {};
|
82 |
|
83 | this.model = new urlModel (this.url, this);
|
84 | this.url = this.model.url;
|
85 |
|
86 | this.model.proxy = this.proxy;
|
87 |
|
88 |
|
89 |
|
90 |
|
91 |
|
92 |
|
93 | this.model.on ('progress', function (current, total) {
|
94 | this.activityCheck ('model.fetch data');
|
95 | this.emit ('progress', current, total);
|
96 | }.bind (this));
|
97 |
|
98 |
|
99 |
|
100 |
|
101 |
|
102 | this.model.on ('error', function (error) {
|
103 |
|
104 | this.clearOperationTimeout();
|
105 | this.finishWith (error, 'failed');
|
106 | }.bind (this));
|
107 |
|
108 | }
|
109 |
|
110 | cacheTask.prototype.isSameUrlLoading = function () {
|
111 | var self = this;
|
112 |
|
113 |
|
114 | var anotherTask = cacheTask.caching[self.cacheFilePath];
|
115 |
|
116 | if (anotherTask && anotherTask != self) {
|
117 |
|
118 | this.emit ('log', 'another process already downloading ' + this.url.href + ' to ' + this.cacheFilePath);
|
119 |
|
120 | anotherTask.on ('complete', function (data) {
|
121 |
|
122 |
|
123 | self.completed (data);
|
124 | });
|
125 | anotherTask.on ('error', function (e, data) {
|
126 | self.emitError(e, data);
|
127 | });
|
128 | return true;
|
129 | } else {
|
130 | cacheTask.caching[self.cacheFilePath] = self;
|
131 | }
|
132 | return false;
|
133 | };
|
134 |
|
135 | cacheTask.prototype.activityCheck = function (place) {
|
136 | var self = this;
|
137 |
|
138 | self.clearOperationTimeout();
|
139 |
|
140 | self.timeoutId = setTimeout(function () {
|
141 | self.state = 5;
|
142 | self.emit (
|
143 | 'log', 'timeout is over for ' + place + ' operation'
|
144 | );
|
145 | self.model.stop ('timeout');
|
146 |
|
147 |
|
148 | }, self.timeout);
|
149 |
|
150 | }
|
151 |
|
152 |
|
153 |
|
154 |
|
155 |
|
156 |
|
157 |
|
158 |
|
159 |
|
160 |
|
161 |
|
162 |
|
163 |
|
164 |
|
165 |
|
166 |
|
167 | cacheTask.prototype.toBuffer = function () {
|
168 | var self = this;
|
169 |
|
170 | self.download = {};
|
171 |
|
172 | self.activityCheck ('task run');
|
173 |
|
174 |
|
175 |
|
176 |
|
177 | if (!self.model) {
|
178 |
|
179 |
|
180 | self.initModel ();
|
181 | self.model.on ('end', function () {
|
182 | |
183 |
|
184 |
|
185 |
|
186 | self.clearOperationTimeout();
|
187 |
|
188 |
|
189 | self.finishWith ({
|
190 | data: self.download.data
|
191 | });
|
192 | });
|
193 |
|
194 |
|
195 | }
|
196 |
|
197 | self.emit ('log', 'start loading from ' + self.url.href + ' to memory buffer');
|
198 |
|
199 | self.activityCheck ('model.fetch start');
|
200 | self.model.fetch ({to: self.download});
|
201 | };
|
202 |
|
203 | cacheTask.prototype.finishWith = function (result, method, metaJSON) {
|
204 | var self = this;
|
205 | var model = self.model,
|
206 | ds;
|
207 |
|
208 | var meta;
|
209 | if (metaJSON) {
|
210 | meta = JSON.parse (metaJSON);
|
211 | }
|
212 |
|
213 | if (model)
|
214 | ds = self.model.dataSource;
|
215 | if (ds && ds.addResultFields) {
|
216 | ds.addResultFields (result, meta);
|
217 | }
|
218 |
|
219 |
|
220 | if (!method)
|
221 | if (ds && ds.isSuccessResponse && ds.isSuccessResponse ()) {
|
222 | method = 'completed';
|
223 | } else {
|
224 | method = 'failed';
|
225 | }
|
226 |
|
227 | self[method] (result);
|
228 | };
|
229 |
|
230 |
|
231 |
|
232 |
|
233 |
|
234 |
|
235 |
|
236 |
|
237 |
|
238 |
|
239 |
|
240 |
|
241 |
|
242 |
|
243 |
|
244 | cacheTask.prototype.toFile = function () {
|
245 | var self = this;
|
246 |
|
247 | self.activityCheck ('task run');
|
248 |
|
249 |
|
250 |
|
251 |
|
252 | if (!self.model) {
|
253 |
|
254 |
|
255 | self.initModel ();
|
256 | self.model.on ('end', function () {
|
257 | |
258 |
|
259 |
|
260 |
|
261 | self.clearOperationTimeout();
|
262 | self.cacheFile.chmod (0640, function (err) {
|
263 |
|
264 | delete cacheTask.caching[self.cacheFilePath];
|
265 |
|
266 |
|
267 | self.finishWith ({
|
268 | fileName: self.cacheFileName,
|
269 | filePath: self.cacheFilePath
|
270 | });
|
271 | });
|
272 | var metaFile = new io (self.cacheFilePath+'.meta');
|
273 | metaFile.writeFile (JSON.stringify ({
|
274 | code: self.model.dataSource.res.statusCode,
|
275 | headers: self.model.dataSource.res.headers,
|
276 | url: self.model.url.href,
|
277 | urlFileName: path.basename (self.model.url.href)
|
278 | }, null, "\t"));
|
279 | });
|
280 | }
|
281 |
|
282 | this.generateCacheFileName ();
|
283 |
|
284 | if (self.isSameUrlLoading ())
|
285 | return;
|
286 |
|
287 | self.cacheFile.stat (function (err, stats) {
|
288 |
|
289 | if (err || ! (stats.mode & 0644 ^ 0600)) {
|
290 | return self.cacheMiss ();
|
291 | }
|
292 |
|
293 | var metaFile = new io (self.cacheFilePath+'.meta');
|
294 | metaFile.readFile (function (err, contents) {
|
295 |
|
296 | if (err) {
|
297 | return self.cacheMiss ();
|
298 | }
|
299 |
|
300 | try {
|
301 | var js = JSON.parse (contents);
|
302 | } catch (e) {
|
303 | return self.cacheMiss ();
|
304 | }
|
305 |
|
306 | self.clearOperationTimeout();
|
307 |
|
308 | self.emit ('log', 'file already downloaded from ' + self.url.href + ' to ' + self.cacheFilePath);
|
309 | delete cacheTask.caching[self.cacheFilePath];
|
310 |
|
311 | self.finishWith ({
|
312 | fileName: self.cacheFileName,
|
313 | filePath: self.cacheFilePath,
|
314 | }, 'completed', contents);
|
315 | });
|
316 | });
|
317 | };
|
318 |
|
319 | cacheTask.prototype.cacheMiss = function () {
|
320 | try {
|
321 | var writeStream = this.cacheFile.writeStream ({
|
322 | flags: 'w',
|
323 | mode: 0600
|
324 | });
|
325 | writeStream.on ('error', function (err) {
|
326 | this.emit ('log', 'write error: ' + err);
|
327 | writeStream.end();
|
328 | });
|
329 | } catch (e) {
|
330 | this.emit ('log', 'cannot open for write: ' + this.cacheFilePath);
|
331 | this.emitError(e);
|
332 | return;
|
333 | }
|
334 |
|
335 | this.emit ('log', 'start caching from ' + this.url.href + ' to ' + this.cacheFilePath);
|
336 |
|
337 | this.activityCheck ('model.fetch start');
|
338 | this.model.fetch ({to: writeStream});
|
339 | }
|
340 |
|
341 | cacheTask.prototype.run = cacheTask.prototype.toFile;
|
342 |
|
343 | cacheTask.prototype.emitError = function (e, data) {
|
344 | if (e) {
|
345 | this.finishWith (data || e, 'failed');
|
346 | return true;
|
347 | } else {
|
348 | return false;
|
349 | }
|
350 | }
|