1 | const fs = require('fs-extra');
|
2 | const path = require('path');
|
3 | const { default: PQueue } = require('p-queue');
|
4 | const {
|
5 | ApiErrorContext,
|
6 | FileSystemErrorContext,
|
7 | logApiErrorInstance,
|
8 | logFileSystemErrorInstance,
|
9 | } = require('./errorHandlers');
|
10 | const { logger } = require('./logger');
|
11 | const {
|
12 | getAllowedExtensions,
|
13 | getCwd,
|
14 | getExt,
|
15 | convertToLocalFileSystemPath,
|
16 | } = require('./path');
|
17 | const { fetchFileStream, download } = require('./api/fileMapper');
|
18 | const {
|
19 | Mode,
|
20 | MODULE_EXTENSION,
|
21 | FUNCTIONS_EXTENSION,
|
22 | } = require('./lib/constants');
|
23 |
|
24 | const queue = new PQueue({
|
25 | concurrency: 10,
|
26 | });
|
27 |
|
28 |
|
29 |
|
30 |
|
31 |
|
32 |
|
33 | function isPathToFile(filepath) {
|
34 | const ext = getExt(filepath);
|
35 | return !!ext && ext !== MODULE_EXTENSION && ext !== FUNCTIONS_EXTENSION;
|
36 | }
|
37 |
|
38 |
|
39 |
|
40 |
|
41 |
|
42 |
|
43 | function isPathToModule(filepath) {
|
44 | const ext = getExt(filepath);
|
45 | return ext === MODULE_EXTENSION;
|
46 | }
|
47 |
|
48 |
|
49 |
|
50 |
|
51 |
|
52 |
|
53 | function isPathToRoot(filepath) {
|
54 | if (typeof filepath !== 'string') return false;
|
55 |
|
56 | return /^(\/|\\)?$/.test(filepath.trim());
|
57 | }
|
58 |
|
59 |
|
60 |
|
61 |
|
62 |
|
63 |
|
64 | function isAllowedExtension(filepath) {
|
65 | const ext = getExt(filepath);
|
66 | if (!ext) return false;
|
67 | return getAllowedExtensions().has(ext);
|
68 | }
|
69 |
|
70 |
|
71 |
|
72 |
|
73 |
|
74 |
|
75 | function useApiBuffer(mode) {
|
76 | return mode === Mode.draft;
|
77 | }
|
78 |
|
79 |
|
80 |
|
81 |
|
82 | function getFileMapperApiQueryFromMode(mode) {
|
83 | return {
|
84 | buffer: useApiBuffer(mode),
|
85 | };
|
86 | }
|
87 |
|
88 |
|
89 |
|
90 |
|
91 |
|
92 |
|
93 |
|
94 |
|
95 |
|
96 |
|
97 |
|
98 |
|
99 |
|
100 |
|
101 |
|
102 |
|
103 |
|
104 |
|
105 |
|
106 |
|
107 | function validateFileMapperNode(node) {
|
108 | if (node === Object(node)) return;
|
109 | let json;
|
110 | try {
|
111 | json = JSON.stringify(node, null, 2);
|
112 | } catch (err) {
|
113 | json = node;
|
114 | }
|
115 | throw new TypeError(`Invalid FileMapperNode: ${json}`);
|
116 | }
|
117 |
|
118 |
|
119 |
|
120 |
|
121 |
|
122 |
|
123 |
|
124 |
|
125 |
|
126 |
|
127 |
|
128 |
|
129 |
|
130 |
|
131 |
|
132 |
|
133 |
|
134 |
|
135 |
|
136 |
|
137 |
|
138 | function getTypeDataFromPath(src) {
|
139 | const isModule = isPathToModule(src);
|
140 | const isFile = !isModule && isPathToFile(src);
|
141 | const isRoot = !isModule && !isFile && isPathToRoot(src);
|
142 | const isFolder = !isFile;
|
143 | return {
|
144 | isModule,
|
145 | isFile,
|
146 | isRoot,
|
147 | isFolder,
|
148 | };
|
149 | }
|
150 |
|
151 |
|
152 |
|
153 |
|
154 |
|
155 |
|
156 |
|
157 |
|
158 |
|
159 |
|
160 |
|
161 |
|
162 |
|
163 |
|
164 |
|
165 |
|
166 |
|
167 |
|
168 | function recurseFolder(node, callback, filepath = '', depth = 0) {
|
169 | validateFileMapperNode(node);
|
170 | const isRootFolder = node.folder && depth === 0;
|
171 | if (isRootFolder) {
|
172 | if (!filepath) {
|
173 | filepath = node.name;
|
174 | }
|
175 | } else {
|
176 | filepath = path.join(filepath, node.name);
|
177 | }
|
178 | let __break = callback(node, { filepath, depth });
|
179 | if (__break === false) return __break;
|
180 | __break = node.children.every(childNode => {
|
181 | __break = recurseFolder(childNode, callback, filepath, depth + 1);
|
182 | return __break !== false;
|
183 | });
|
184 | return depth === 0 ? undefined : __break;
|
185 | }
|
186 |
|
187 |
|
188 |
|
189 |
|
190 |
|
191 |
|
192 |
|
193 |
|
194 |
|
195 | async function writeUtimes(input, filepath, node) {
|
196 | try {
|
197 | const now = new Date();
|
198 | const atime = node.createdAt ? new Date(node.createdAt) : now;
|
199 | const mtime = node.updatedAt ? new Date(node.updatedAt) : now;
|
200 | await fs.utimes(filepath, atime, mtime);
|
201 | } catch (err) {
|
202 | logFileSystemErrorInstance(
|
203 | err,
|
204 | new FileSystemErrorContext({
|
205 | filepath,
|
206 | portalId: input.portalId,
|
207 | write: true,
|
208 | })
|
209 | );
|
210 | }
|
211 | }
|
212 |
|
213 |
|
214 |
|
215 |
|
216 |
|
217 |
|
218 |
|
219 |
|
220 | async function skipExisting(input, filepath) {
|
221 | if (input.options.overwrite) {
|
222 | return false;
|
223 | }
|
224 | if (await fs.pathExists(filepath)) {
|
225 | logger.log('Skipped existing "%s"', filepath);
|
226 | return true;
|
227 | }
|
228 | return false;
|
229 | }
|
230 |
|
231 |
|
232 |
|
233 |
|
234 |
|
235 |
|
236 |
|
237 |
|
238 | async function fetchAndWriteFileStream(input, srcPath, filepath) {
|
239 | if (await skipExisting(input, filepath)) {
|
240 | return null;
|
241 | }
|
242 | if (!isAllowedExtension(srcPath)) {
|
243 | const message = `Invalid file type requested: "${srcPath}"`;
|
244 | logger.error(message);
|
245 | throw new Error(message);
|
246 | }
|
247 | const { portalId } = input;
|
248 | const logFsError = err => {
|
249 | logFileSystemErrorInstance(
|
250 | err,
|
251 | new FileSystemErrorContext({
|
252 | filepath,
|
253 | portalId,
|
254 | write: true,
|
255 | })
|
256 | );
|
257 | };
|
258 | let writeStream;
|
259 | try {
|
260 | await fs.ensureFile(filepath);
|
261 | writeStream = fs.createWriteStream(filepath, { encoding: 'binary' });
|
262 | } catch (err) {
|
263 | logFsError(err);
|
264 | throw err;
|
265 | }
|
266 | let node;
|
267 | try {
|
268 | node = await fetchFileStream(portalId, srcPath, writeStream, {
|
269 | qs: getFileMapperApiQueryFromMode(input.mode),
|
270 | });
|
271 | } catch (err) {
|
272 | logApiErrorInstance(
|
273 | err,
|
274 | new ApiErrorContext({
|
275 | portalId,
|
276 | request: srcPath,
|
277 | })
|
278 | );
|
279 | throw err;
|
280 | }
|
281 | return new Promise((resolve, reject) => {
|
282 | writeStream.on('error', err => {
|
283 | logFsError(err);
|
284 | reject(err);
|
285 | });
|
286 | writeStream.on('close', async () => {
|
287 | await writeUtimes(input, filepath, node);
|
288 | logger.log('Wrote file "%s"', filepath);
|
289 | resolve(node);
|
290 | });
|
291 | });
|
292 | }
|
293 |
|
294 |
|
295 |
|
296 |
|
297 |
|
298 |
|
299 |
|
300 |
|
301 |
|
302 |
|
303 |
|
304 |
|
305 | async function writeFileMapperNode(input, node, filepath) {
|
306 | filepath = convertToLocalFileSystemPath(path.resolve(filepath));
|
307 | if (await skipExisting(input, filepath)) {
|
308 | return;
|
309 | }
|
310 | if (!node.folder) {
|
311 | try {
|
312 | await fetchAndWriteFileStream(input, node.path, filepath);
|
313 | } catch (err) {
|
314 |
|
315 | }
|
316 | return;
|
317 | }
|
318 | try {
|
319 | await fs.ensureDir(filepath);
|
320 | logger.log('Wrote folder "%s"', filepath);
|
321 | } catch (err) {
|
322 | logFileSystemErrorInstance(
|
323 | err,
|
324 | new FileSystemErrorContext({
|
325 | filepath,
|
326 | portalId: input.portalId,
|
327 | write: true,
|
328 | })
|
329 | );
|
330 | }
|
331 | }
|
332 |
|
333 |
|
334 |
|
335 |
|
336 |
|
337 |
|
338 |
|
339 | async function downloadFile(input) {
|
340 | try {
|
341 | const { src } = input;
|
342 | const { isFile } = getTypeDataFromPath(src);
|
343 | if (!isFile) {
|
344 | throw new Error(`Invalid request for file: "${src}"`);
|
345 | }
|
346 | const dest = path.resolve(input.dest);
|
347 | const cwd = getCwd();
|
348 | let filepath;
|
349 | if (dest === cwd) {
|
350 |
|
351 | filepath = path.resolve(cwd, path.basename(src));
|
352 | } else if (isPathToFile(dest)) {
|
353 |
|
354 | filepath = path.isAbsolute(dest) ? dest : path.resolve(cwd, dest);
|
355 | } else {
|
356 |
|
357 | const name = path.basename(src);
|
358 | filepath = path.isAbsolute(dest)
|
359 | ? path.resolve(dest, name)
|
360 | : path.resolve(cwd, dest, name);
|
361 | }
|
362 | const localFsPath = convertToLocalFileSystemPath(filepath);
|
363 | await fetchAndWriteFileStream(input, input.src, localFsPath);
|
364 | await queue.onIdle();
|
365 | logger.log('Completed fetch of file "%s" to "%s"', input.src, localFsPath);
|
366 | } catch (err) {
|
367 | logger.error('Failed fetch of file "%s" to "%s"', input.src, input.dest);
|
368 | }
|
369 | }
|
370 |
|
371 |
|
372 |
|
373 |
|
374 |
|
375 |
|
376 |
|
377 | async function fetchFolderFromApi(input) {
|
378 | const { portalId, src, mode } = input;
|
379 | const { isRoot, isFolder } = getTypeDataFromPath(src);
|
380 | if (!isFolder) {
|
381 | throw new Error(`Invalid request for folder: "${src}"`);
|
382 | }
|
383 | try {
|
384 | const srcPath = isRoot ? '@root' : src;
|
385 | const node = await download(portalId, srcPath, {
|
386 | qs: getFileMapperApiQueryFromMode(mode),
|
387 | });
|
388 | logger.log('Fetched "%s" from portal %d successfully', src, portalId);
|
389 | return node;
|
390 | } catch (err) {
|
391 | logApiErrorInstance(
|
392 | err,
|
393 | new ApiErrorContext({
|
394 | portalId,
|
395 | request: src,
|
396 | })
|
397 | );
|
398 | }
|
399 | return null;
|
400 | }
|
401 |
|
402 |
|
403 |
|
404 |
|
405 |
|
406 |
|
407 |
|
408 | async function downloadFolder(input) {
|
409 | try {
|
410 | const node = await fetchFolderFromApi(input);
|
411 | if (!node) {
|
412 | return;
|
413 | }
|
414 | const dest = path.resolve(input.dest);
|
415 | const rootPath =
|
416 | dest === getCwd()
|
417 | ? convertToLocalFileSystemPath(path.resolve(dest, node.name))
|
418 | : dest;
|
419 | recurseFolder(
|
420 | node,
|
421 | (childNode, { filepath }) => {
|
422 | queue.add(() => writeFileMapperNode(input, childNode, filepath));
|
423 | },
|
424 | rootPath
|
425 | );
|
426 | await queue.onIdle();
|
427 | logger.log('Completed fetch of folder "%s" to "%s"', input.src, input.dest);
|
428 | } catch (err) {
|
429 | logger.error('Failed fetch of folder "%s" to "%s"', input.src, input.dest);
|
430 | }
|
431 | }
|
432 |
|
433 |
|
434 |
|
435 |
|
436 |
|
437 |
|
438 |
|
439 |
|
440 | async function downloadFileOrFolder(input) {
|
441 | try {
|
442 | if (!(input && input.src)) {
|
443 | return;
|
444 | }
|
445 | const { isFile } = getTypeDataFromPath(input.src);
|
446 | if (isFile) {
|
447 | await downloadFile(input);
|
448 | } else {
|
449 | await downloadFolder(input);
|
450 | }
|
451 | } catch (err) {
|
452 |
|
453 | }
|
454 | }
|
455 |
|
456 | module.exports = {
|
457 | isPathToFile,
|
458 | isPathToModule,
|
459 | isPathToRoot,
|
460 | downloadFileOrFolder,
|
461 | recurseFolder,
|
462 | getFileMapperApiQueryFromMode,
|
463 | fetchFolderFromApi,
|
464 | getTypeDataFromPath,
|
465 | };
|