1 |
|
2 |
|
3 |
|
4 |
|
5 | "use strict";
|
6 |
|
7 | const { constants } = require("buffer");
|
8 | const createHash = require("../util/createHash");
|
9 | const { dirname, join, mkdirp } = require("../util/fs");
|
10 | const memoize = require("../util/memoize");
|
11 | const SerializerMiddleware = require("./SerializerMiddleware");
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 |
|
18 |
|
19 |
|
20 |
|
21 |
|
22 |
|
23 |
|
24 |
|
25 |
|
26 |
|
27 |
|
28 |
|
29 |
|
30 |
|
31 |
|
32 |
|
33 | const VERSION = 0x01637077;
|
34 | const hashForName = buffers => {
|
35 | const hash = createHash("md4");
|
36 | for (const buf of buffers) hash.update(buf);
|
37 | return (hash.digest("hex"));
|
38 | };
|
39 |
|
40 | const writeUInt64LE = Buffer.prototype.writeBigUInt64LE
|
41 | ? (buf, value, offset) => {
|
42 | buf.writeBigUInt64LE(BigInt(value), offset);
|
43 | }
|
44 | : (buf, value, offset) => {
|
45 | const low = value % 0x100000000;
|
46 | const high = (value - low) / 0x100000000;
|
47 | buf.writeUInt32LE(low, offset);
|
48 | buf.writeUInt32LE(high, offset + 4);
|
49 | };
|
50 |
|
51 | const readUInt64LE = Buffer.prototype.readBigUInt64LE
|
52 | ? (buf, offset) => {
|
53 | return Number(buf.readBigUInt64LE(offset));
|
54 | }
|
55 | : (buf, offset) => {
|
56 | const low = buf.readUInt32LE(offset);
|
57 | const high = buf.readUInt32LE(offset + 4);
|
58 | return high * 0x100000000 + low;
|
59 | };
|
60 |
|
61 |
|
62 |
|
63 |
|
64 |
|
65 |
|
66 |
|
67 |
|
68 |
|
69 |
|
70 |
|
71 |
|
72 |
|
73 |
|
74 |
|
75 | const serialize = async (middleware, data, name, writeFile) => {
|
76 |
|
77 | const processedData = [];
|
78 |
|
79 | const resultToLazy = new WeakMap();
|
80 |
|
81 | let lastBuffers = undefined;
|
82 | for (const item of await data) {
|
83 | if (typeof item === "function") {
|
84 | if (!SerializerMiddleware.isLazy(item))
|
85 | throw new Error("Unexpected function");
|
86 | if (!SerializerMiddleware.isLazy(item, middleware)) {
|
87 | throw new Error(
|
88 | "Unexpected lazy value with non-this target (can't pass through lazy values)"
|
89 | );
|
90 | }
|
91 | lastBuffers = undefined;
|
92 | const serializedInfo = SerializerMiddleware.getLazySerializedValue(item);
|
93 | if (serializedInfo) {
|
94 | if (typeof serializedInfo === "function") {
|
95 | throw new Error(
|
96 | "Unexpected lazy value with non-this target (can't pass through lazy values)"
|
97 | );
|
98 | } else {
|
99 | processedData.push(serializedInfo);
|
100 | }
|
101 | } else {
|
102 | const content = item();
|
103 | if (content) {
|
104 | const options = SerializerMiddleware.getLazyOptions(item);
|
105 | processedData.push(
|
106 | serialize(
|
107 | middleware,
|
108 | content,
|
109 | (options && options.name) || true,
|
110 | writeFile
|
111 | ).then(result => {
|
112 | (item).options.size = result.size;
|
113 | resultToLazy.set(result, item);
|
114 | return result;
|
115 | })
|
116 | );
|
117 | } else {
|
118 | throw new Error(
|
119 | "Unexpected falsy value returned by lazy value function"
|
120 | );
|
121 | }
|
122 | }
|
123 | } else if (item) {
|
124 | if (lastBuffers) {
|
125 | lastBuffers.push(item);
|
126 | } else {
|
127 | lastBuffers = [item];
|
128 | processedData.push(lastBuffers);
|
129 | }
|
130 | } else {
|
131 | throw new Error("Unexpected falsy value in items array");
|
132 | }
|
133 | }
|
134 |
|
135 | const backgroundJobs = [];
|
136 | const resolvedData = (
|
137 | await Promise.all(
|
138 | (processedData)
|
139 | )
|
140 | ).map(item => {
|
141 | if (Array.isArray(item) || Buffer.isBuffer(item)) return item;
|
142 |
|
143 | backgroundJobs.push(item.backgroundJob);
|
144 |
|
145 | const name = (item.name);
|
146 | const nameBuffer = Buffer.from(name);
|
147 | const buf = Buffer.allocUnsafe(8 + nameBuffer.length);
|
148 | writeUInt64LE(buf, item.size, 0);
|
149 | nameBuffer.copy(buf, 8, 0);
|
150 | const lazy = resultToLazy.get(item);
|
151 | SerializerMiddleware.setLazySerializedValue(lazy, buf);
|
152 | return buf;
|
153 | });
|
154 | const lengths = [];
|
155 | for (const item of resolvedData) {
|
156 | if (Array.isArray(item)) {
|
157 | let l = 0;
|
158 | for (const b of item) l += b.length;
|
159 | while (l > 0x7fffffff) {
|
160 | lengths.push(0x7fffffff);
|
161 | l -= 0x7fffffff;
|
162 | }
|
163 | lengths.push(l);
|
164 | } else if (item) {
|
165 | lengths.push(-item.length);
|
166 | } else {
|
167 | throw new Error("Unexpected falsy value in resolved data " + item);
|
168 | }
|
169 | }
|
170 | const header = Buffer.allocUnsafe(8 + lengths.length * 4);
|
171 | header.writeUInt32LE(VERSION, 0);
|
172 | header.writeUInt32LE(lengths.length, 4);
|
173 | for (let i = 0; i < lengths.length; i++) {
|
174 | header.writeInt32LE(lengths[i], 8 + i * 4);
|
175 | }
|
176 | const buf = [header];
|
177 | for (const item of resolvedData) {
|
178 | if (Array.isArray(item)) {
|
179 | for (const b of item) buf.push(b);
|
180 | } else if (item) {
|
181 | buf.push(item);
|
182 | }
|
183 | }
|
184 | if (name === true) {
|
185 | name = hashForName(buf);
|
186 | }
|
187 | backgroundJobs.push(writeFile(name, buf));
|
188 | let size = 0;
|
189 | for (const b of buf) size += b.length;
|
190 | return {
|
191 | size,
|
192 | name,
|
193 | backgroundJob:
|
194 | backgroundJobs.length === 1
|
195 | ? backgroundJobs[0]
|
196 | : Promise.all(backgroundJobs)
|
197 | };
|
198 | };
|
199 |
|
200 |
|
201 |
|
202 |
|
203 |
|
204 |
|
205 |
|
206 | const deserialize = async (middleware, name, readFile) => {
|
207 | const contents = await readFile(name);
|
208 | if (contents.length === 0) throw new Error("Empty file " + name);
|
209 | let contentsIndex = 0;
|
210 | let contentItem = contents[0];
|
211 | let contentItemLength = contentItem.length;
|
212 | let contentPosition = 0;
|
213 | if (contentItemLength === 0) throw new Error("Empty file " + name);
|
214 | const nextContent = () => {
|
215 | contentsIndex++;
|
216 | contentItem = contents[contentsIndex];
|
217 | contentItemLength = contentItem.length;
|
218 | contentPosition = 0;
|
219 | };
|
220 | const ensureData = n => {
|
221 | if (contentPosition === contentItemLength) {
|
222 | nextContent();
|
223 | }
|
224 | while (contentItemLength - contentPosition < n) {
|
225 | const remaining = contentItem.slice(contentPosition);
|
226 | let lengthFromNext = n - remaining.length;
|
227 | const buffers = [remaining];
|
228 | for (let i = contentsIndex + 1; i < contents.length; i++) {
|
229 | const l = contents[i].length;
|
230 | if (l > lengthFromNext) {
|
231 | buffers.push(contents[i].slice(0, lengthFromNext));
|
232 | contents[i] = contents[i].slice(lengthFromNext);
|
233 | lengthFromNext = 0;
|
234 | break;
|
235 | } else {
|
236 | buffers.push(contents[i]);
|
237 | contentsIndex = i;
|
238 | lengthFromNext -= l;
|
239 | }
|
240 | }
|
241 | if (lengthFromNext > 0) throw new Error("Unexpected end of data");
|
242 | contentItem = Buffer.concat(buffers, n);
|
243 | contentItemLength = n;
|
244 | contentPosition = 0;
|
245 | }
|
246 | };
|
247 | const readUInt32LE = () => {
|
248 | ensureData(4);
|
249 | const value = contentItem.readUInt32LE(contentPosition);
|
250 | contentPosition += 4;
|
251 | return value;
|
252 | };
|
253 | const readInt32LE = () => {
|
254 | ensureData(4);
|
255 | const value = contentItem.readInt32LE(contentPosition);
|
256 | contentPosition += 4;
|
257 | return value;
|
258 | };
|
259 | const readSlice = l => {
|
260 | ensureData(l);
|
261 | if (contentPosition === 0 && contentItemLength === l) {
|
262 | const result = contentItem;
|
263 | if (contentsIndex + 1 < contents.length) {
|
264 | nextContent();
|
265 | } else {
|
266 | contentPosition = l;
|
267 | }
|
268 | return result;
|
269 | }
|
270 | const result = contentItem.slice(contentPosition, contentPosition + l);
|
271 | contentPosition += l;
|
272 |
|
273 | return l * 2 < contentItem.buffer.byteLength ? Buffer.from(result) : result;
|
274 | };
|
275 | const version = readUInt32LE();
|
276 | if (version !== VERSION) {
|
277 | throw new Error("Invalid file version");
|
278 | }
|
279 | const sectionCount = readUInt32LE();
|
280 | const lengths = [];
|
281 | for (let i = 0; i < sectionCount; i++) {
|
282 | lengths.push(readInt32LE());
|
283 | }
|
284 | const result = [];
|
285 | for (let length of lengths) {
|
286 | if (length < 0) {
|
287 | const slice = readSlice(-length);
|
288 | const size = Number(readUInt64LE(slice, 0));
|
289 | const nameBuffer = slice.slice(8);
|
290 | const name = nameBuffer.toString();
|
291 | result.push(
|
292 | SerializerMiddleware.createLazy(
|
293 | memoize(() => deserialize(middleware, name, readFile)),
|
294 | middleware,
|
295 | {
|
296 | name,
|
297 | size
|
298 | },
|
299 | slice
|
300 | )
|
301 | );
|
302 | } else {
|
303 | if (contentPosition === contentItemLength) {
|
304 | nextContent();
|
305 | } else if (contentPosition !== 0) {
|
306 | if (length <= contentItemLength - contentPosition) {
|
307 | result.push(
|
308 | contentItem.slice(contentPosition, contentPosition + length)
|
309 | );
|
310 | contentPosition += length;
|
311 | length = 0;
|
312 | } else {
|
313 | result.push(contentItem.slice(contentPosition));
|
314 | length -= contentItemLength - contentPosition;
|
315 | contentPosition = contentItemLength;
|
316 | }
|
317 | } else {
|
318 | if (length >= contentItemLength) {
|
319 | result.push(contentItem);
|
320 | length -= contentItemLength;
|
321 | contentPosition = contentItemLength;
|
322 | } else {
|
323 | result.push(contentItem.slice(0, length));
|
324 | contentPosition += length;
|
325 | length = 0;
|
326 | }
|
327 | }
|
328 | while (length > 0) {
|
329 | nextContent();
|
330 | if (length >= contentItemLength) {
|
331 | result.push(contentItem);
|
332 | length -= contentItemLength;
|
333 | contentPosition = contentItemLength;
|
334 | } else {
|
335 | result.push(contentItem.slice(0, length));
|
336 | contentPosition += length;
|
337 | length = 0;
|
338 | }
|
339 | }
|
340 | }
|
341 | }
|
342 | return result;
|
343 | };
|
344 |
|
345 |
|
346 |
|
347 |
|
348 |
|
349 |
|
350 | class FileMiddleware extends SerializerMiddleware {
|
351 | |
352 |
|
353 |
|
354 | constructor(fs) {
|
355 | super();
|
356 | this.fs = fs;
|
357 | }
|
358 | |
359 |
|
360 |
|
361 |
|
362 |
|
363 | serialize(data, context) {
|
364 | const { filename, extension = "" } = context;
|
365 | return new Promise((resolve, reject) => {
|
366 | mkdirp(this.fs, dirname(this.fs, filename), err => {
|
367 | if (err) return reject(err);
|
368 |
|
369 |
|
370 |
|
371 | const allWrittenFiles = new Set();
|
372 | const writeFile = async (name, content) => {
|
373 | const file = name
|
374 | ? join(this.fs, filename, `../${name}${extension}`)
|
375 | : filename;
|
376 | await new Promise((resolve, reject) => {
|
377 | const stream = this.fs.createWriteStream(file + "_");
|
378 | for (const b of content) stream.write(b);
|
379 | stream.end();
|
380 | stream.on("error", err => reject(err));
|
381 | stream.on("finish", () => resolve());
|
382 | });
|
383 | if (name) allWrittenFiles.add(file);
|
384 | };
|
385 |
|
386 | resolve(
|
387 | serialize(this, data, false, writeFile).then(
|
388 | async ({ backgroundJob }) => {
|
389 | await backgroundJob;
|
390 |
|
391 |
|
392 | await new Promise(resolve =>
|
393 | this.fs.rename(filename, filename + ".old", err => {
|
394 | resolve();
|
395 | })
|
396 | );
|
397 |
|
398 |
|
399 | await Promise.all(
|
400 | Array.from(
|
401 | allWrittenFiles,
|
402 | file =>
|
403 | new Promise((resolve, reject) => {
|
404 | this.fs.rename(file + "_", file, err => {
|
405 | if (err) return reject(err);
|
406 | resolve();
|
407 | });
|
408 | })
|
409 | )
|
410 | );
|
411 |
|
412 |
|
413 | await new Promise(resolve => {
|
414 | this.fs.rename(filename + "_", filename, err => {
|
415 | if (err) return reject(err);
|
416 | resolve();
|
417 | });
|
418 | });
|
419 | return (true);
|
420 | }
|
421 | )
|
422 | );
|
423 | });
|
424 | });
|
425 | }
|
426 |
|
427 | |
428 |
|
429 |
|
430 |
|
431 |
|
432 | deserialize(data, context) {
|
433 | const { filename, extension = "" } = context;
|
434 | const readFile = name =>
|
435 | new Promise((resolve, reject) => {
|
436 | const file = name
|
437 | ? join(this.fs, filename, `../${name}${extension}`)
|
438 | : filename;
|
439 | this.fs.stat(file, (err, stats) => {
|
440 | if (err) {
|
441 | reject(err);
|
442 | return;
|
443 | }
|
444 | let remaining = (stats.size);
|
445 | let currentBuffer;
|
446 | let currentBufferUsed;
|
447 | const buf = [];
|
448 | this.fs.open(file, "r", (err, fd) => {
|
449 | if (err) {
|
450 | reject(err);
|
451 | return;
|
452 | }
|
453 | const read = () => {
|
454 | if (currentBuffer === undefined) {
|
455 | currentBuffer = Buffer.allocUnsafeSlow(
|
456 | Math.min(constants.MAX_LENGTH, remaining)
|
457 | );
|
458 | currentBufferUsed = 0;
|
459 | }
|
460 | let readBuffer = currentBuffer;
|
461 | let readOffset = currentBufferUsed;
|
462 | let readLength = currentBuffer.length - currentBufferUsed;
|
463 | if (readOffset > 0x7fffffff) {
|
464 | readBuffer = currentBuffer.slice(readOffset);
|
465 | readOffset = 0;
|
466 | }
|
467 | if (readLength > 0x7fffffff) {
|
468 | readLength = 0x7fffffff;
|
469 | }
|
470 | this.fs.read(
|
471 | fd,
|
472 | readBuffer,
|
473 | readOffset,
|
474 | readLength,
|
475 | null,
|
476 | (err, bytesRead) => {
|
477 | if (err) {
|
478 | this.fs.close(fd, () => {
|
479 | reject(err);
|
480 | });
|
481 | return;
|
482 | }
|
483 | currentBufferUsed += bytesRead;
|
484 | remaining -= bytesRead;
|
485 | if (currentBufferUsed === currentBuffer.length) {
|
486 | buf.push(currentBuffer);
|
487 | currentBuffer = undefined;
|
488 | if (remaining === 0) {
|
489 | this.fs.close(fd, err => {
|
490 | if (err) {
|
491 | reject(err);
|
492 | return;
|
493 | }
|
494 | resolve(buf);
|
495 | });
|
496 | return;
|
497 | }
|
498 | }
|
499 | read();
|
500 | }
|
501 | );
|
502 | };
|
503 | read();
|
504 | });
|
505 | });
|
506 | });
|
507 | return deserialize(this, false, readFile);
|
508 | }
|
509 | }
|
510 |
|
511 | module.exports = FileMiddleware;
|