1 | "use strict";
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 | var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
18 | if (k2 === undefined) k2 = k;
|
19 | var desc = Object.getOwnPropertyDescriptor(m, k);
|
20 | if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
21 | desc = { enumerable: true, get: function() { return m[k]; } };
|
22 | }
|
23 | Object.defineProperty(o, k2, desc);
|
24 | }) : (function(o, m, k, k2) {
|
25 | if (k2 === undefined) k2 = k;
|
26 | o[k2] = m[k];
|
27 | }));
|
28 | var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
29 | Object.defineProperty(o, "default", { enumerable: true, value: v });
|
30 | }) : function(o, v) {
|
31 | o["default"] = v;
|
32 | });
|
33 | var __importStar = (this && this.__importStar) || function (mod) {
|
34 | if (mod && mod.__esModule) return mod;
|
35 | var result = {};
|
36 | if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
37 | __setModuleDefault(result, mod);
|
38 | return result;
|
39 | };
|
40 | var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
|
41 | if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
|
42 | if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
43 | return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
44 | };
|
45 | var __importDefault = (this && this.__importDefault) || function (mod) {
|
46 | return (mod && mod.__esModule) ? mod : { "default": mod };
|
47 | };
|
48 | var _XMLMultiPartUploadHelper_instances, _XMLMultiPartUploadHelper_setGoogApiClientHeaders, _XMLMultiPartUploadHelper_handleErrorResponse;
|
49 | Object.defineProperty(exports, "__esModule", { value: true });
|
50 | exports.TransferManager = exports.MultiPartUploadError = void 0;
|
51 | const file_js_1 = require("./file.js");
|
52 | const p_limit_1 = __importDefault(require("p-limit"));
|
53 | const path = __importStar(require("path"));
|
54 | const fs_1 = require("fs");
|
55 | const crc32c_js_1 = require("./crc32c.js");
|
56 | const google_auth_library_1 = require("google-auth-library");
|
57 | const fast_xml_parser_1 = require("fast-xml-parser");
|
58 | const async_retry_1 = __importDefault(require("async-retry"));
|
59 | const crypto_1 = require("crypto");
|
60 | const util_js_1 = require("./nodejs-common/util.js");
|
61 | const util_js_2 = require("./util.js");
|
62 |
|
63 |
|
64 | const package_json_helper_cjs_1 = require("./package-json-helper.cjs");
|
65 | const packageJson = (0, package_json_helper_cjs_1.getPackageJSON)();
|
66 |
|
67 |
|
68 |
|
69 |
|
70 | const DEFAULT_PARALLEL_UPLOAD_LIMIT = 5;
|
71 |
|
72 |
|
73 |
|
74 |
|
75 | const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 5;
|
76 |
|
77 |
|
78 |
|
79 |
|
80 | const DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT = 5;
|
81 |
|
82 |
|
83 |
|
84 |
|
85 | const DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD = 32 * 1024 * 1024;
|
86 |
|
87 |
|
88 |
|
89 |
|
90 | const DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024;
|
91 |
|
92 |
|
93 |
|
94 |
|
95 | const UPLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024;
|
96 |
|
97 |
|
98 |
|
99 |
|
100 | const DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT = 5;
|
101 | const EMPTY_REGEX = '(?:)';
|
102 |
|
103 |
|
104 |
|
105 |
|
106 |
|
107 |
|
108 |
|
109 | const GCCL_GCS_CMD_FEATURE = {
|
110 | UPLOAD_MANY: 'tm.upload_many',
|
111 | DOWNLOAD_MANY: 'tm.download_many',
|
112 | UPLOAD_SHARDED: 'tm.upload_sharded',
|
113 | DOWNLOAD_SHARDED: 'tm.download_sharded',
|
114 | };
|
115 | const defaultMultiPartGenerator = (bucket, fileName, uploadId, partsMap) => {
|
116 | return new XMLMultiPartUploadHelper(bucket, fileName, uploadId, partsMap);
|
117 | };
|
118 | class MultiPartUploadError extends Error {
|
119 | constructor(message, uploadId, partsMap) {
|
120 | super(message);
|
121 | this.uploadId = uploadId;
|
122 | this.partsMap = partsMap;
|
123 | }
|
124 | }
|
125 | exports.MultiPartUploadError = MultiPartUploadError;
|
126 |
|
127 |
|
128 |
|
129 |
|
130 |
|
131 |
|
132 | class XMLMultiPartUploadHelper {
|
133 | constructor(bucket, fileName, uploadId, partsMap) {
|
134 | _XMLMultiPartUploadHelper_instances.add(this);
|
135 | this.authClient = bucket.storage.authClient || new google_auth_library_1.GoogleAuth();
|
136 | this.uploadId = uploadId || '';
|
137 | this.bucket = bucket;
|
138 | this.fileName = fileName;
|
139 | this.baseUrl = `https://${bucket.name}.${new URL(this.bucket.storage.apiEndpoint).hostname}/${fileName}`;
|
140 | this.xmlBuilder = new fast_xml_parser_1.XMLBuilder({ arrayNodeName: 'Part' });
|
141 | this.xmlParser = new fast_xml_parser_1.XMLParser();
|
142 | this.partsMap = partsMap || new Map();
|
143 | this.retryOptions = {
|
144 | retries: this.bucket.storage.retryOptions.maxRetries,
|
145 | factor: this.bucket.storage.retryOptions.retryDelayMultiplier,
|
146 | maxTimeout: this.bucket.storage.retryOptions.maxRetryDelay * 1000,
|
147 | maxRetryTime: this.bucket.storage.retryOptions.totalTimeout * 1000,
|
148 | };
|
149 | }
|
150 | |
151 |
|
152 |
|
153 |
|
154 |
|
155 | async initiateUpload(headers = {}) {
|
156 | const url = `${this.baseUrl}?uploads`;
|
157 | return (0, async_retry_1.default)(async (bail) => {
|
158 | try {
|
159 | const res = await this.authClient.request({
|
160 | headers: __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this, headers),
|
161 | method: 'POST',
|
162 | url,
|
163 | });
|
164 | if (res.data && res.data.error) {
|
165 | throw res.data.error;
|
166 | }
|
167 | const parsedXML = this.xmlParser.parse(res.data);
|
168 | this.uploadId = parsedXML.InitiateMultipartUploadResult.UploadId;
|
169 | }
|
170 | catch (e) {
|
171 | __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
|
172 | }
|
173 | }, this.retryOptions);
|
174 | }
|
175 | |
176 |
|
177 |
|
178 |
|
179 |
|
180 |
|
181 |
|
182 |
|
183 |
|
184 | async uploadPart(partNumber, chunk, validation) {
|
185 | const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`;
|
186 | let headers = __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this);
|
187 | if (validation === 'md5') {
|
188 | const hash = (0, crypto_1.createHash)('md5').update(chunk).digest('base64');
|
189 | headers = {
|
190 | 'Content-MD5': hash,
|
191 | };
|
192 | }
|
193 | return (0, async_retry_1.default)(async (bail) => {
|
194 | try {
|
195 | const res = await this.authClient.request({
|
196 | url,
|
197 | method: 'PUT',
|
198 | body: chunk,
|
199 | headers,
|
200 | });
|
201 | if (res.data && res.data.error) {
|
202 | throw res.data.error;
|
203 | }
|
204 | this.partsMap.set(partNumber, res.headers['etag']);
|
205 | }
|
206 | catch (e) {
|
207 | __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
|
208 | }
|
209 | }, this.retryOptions);
|
210 | }
|
211 | |
212 |
|
213 |
|
214 |
|
215 |
|
216 | async completeUpload() {
|
217 | const url = `${this.baseUrl}?uploadId=${this.uploadId}`;
|
218 | const sortedMap = new Map([...this.partsMap.entries()].sort((a, b) => a[0] - b[0]));
|
219 | const parts = [];
|
220 | for (const entry of sortedMap.entries()) {
|
221 | parts.push({ PartNumber: entry[0], ETag: entry[1] });
|
222 | }
|
223 | const body = `<CompleteMultipartUpload>${this.xmlBuilder.build(parts)}</CompleteMultipartUpload>`;
|
224 | return (0, async_retry_1.default)(async (bail) => {
|
225 | try {
|
226 | const res = await this.authClient.request({
|
227 | headers: __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this),
|
228 | url,
|
229 | method: 'POST',
|
230 | body,
|
231 | });
|
232 | if (res.data && res.data.error) {
|
233 | throw res.data.error;
|
234 | }
|
235 | return res;
|
236 | }
|
237 | catch (e) {
|
238 | __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
|
239 | return;
|
240 | }
|
241 | }, this.retryOptions);
|
242 | }
|
243 | |
244 |
|
245 |
|
246 |
|
247 |
|
248 |
|
249 | async abortUpload() {
|
250 | const url = `${this.baseUrl}?uploadId=${this.uploadId}`;
|
251 | return (0, async_retry_1.default)(async (bail) => {
|
252 | try {
|
253 | const res = await this.authClient.request({
|
254 | url,
|
255 | method: 'DELETE',
|
256 | });
|
257 | if (res.data && res.data.error) {
|
258 | throw res.data.error;
|
259 | }
|
260 | }
|
261 | catch (e) {
|
262 | __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
|
263 | return;
|
264 | }
|
265 | }, this.retryOptions);
|
266 | }
|
267 | }
|
268 | _XMLMultiPartUploadHelper_instances = new WeakSet(), _XMLMultiPartUploadHelper_setGoogApiClientHeaders = function _XMLMultiPartUploadHelper_setGoogApiClientHeaders(headers = {}) {
|
269 | let headerFound = false;
|
270 | let userAgentFound = false;
|
271 | for (const [key, value] of Object.entries(headers)) {
|
272 | if (key.toLocaleLowerCase().trim() === 'x-goog-api-client') {
|
273 | headerFound = true;
|
274 |
|
275 | if (!value.includes(GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED)) {
|
276 | headers[key] =
|
277 | `${value} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`;
|
278 | }
|
279 | }
|
280 | else if (key.toLocaleLowerCase().trim() === 'user-agent') {
|
281 | userAgentFound = true;
|
282 | }
|
283 | }
|
284 |
|
285 | if (!headerFound) {
|
286 | headers['x-goog-api-client'] = `${(0, util_js_2.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`;
|
287 | }
|
288 |
|
289 | if (!userAgentFound) {
|
290 | headers['User-Agent'] = (0, util_js_2.getUserAgentString)();
|
291 | }
|
292 | return headers;
|
293 | }, _XMLMultiPartUploadHelper_handleErrorResponse = function _XMLMultiPartUploadHelper_handleErrorResponse(err, bail) {
|
294 | if (this.bucket.storage.retryOptions.autoRetry &&
|
295 | this.bucket.storage.retryOptions.retryableErrorFn(err)) {
|
296 | throw err;
|
297 | }
|
298 | else {
|
299 | bail(err);
|
300 | }
|
301 | };
|
302 |
|
303 |
|
304 |
|
305 |
|
306 |
|
307 |
|
308 |
|
309 |
|
310 |
|
311 | class TransferManager {
|
312 | constructor(bucket) {
|
313 | this.bucket = bucket;
|
314 | }
|
315 | |
316 |
|
317 |
|
318 |
|
319 |
|
320 |
|
321 |
|
322 |
|
323 |
|
324 |
|
325 |
|
326 |
|
327 |
|
328 | |
329 |
|
330 |
|
331 |
|
332 |
|
333 |
|
334 |
|
335 |
|
336 |
|
337 |
|
338 |
|
339 |
|
340 |
|
341 |
|
342 |
|
343 |
|
344 |
|
345 |
|
346 |
|
347 |
|
348 |
|
349 |
|
350 |
|
351 |
|
352 |
|
353 |
|
354 |
|
355 |
|
356 |
|
357 | async uploadManyFiles(filePathsOrDirectory, options = {}) {
|
358 | var _a;
|
359 | if (options.skipIfExists && ((_a = options.passthroughOptions) === null || _a === void 0 ? void 0 : _a.preconditionOpts)) {
|
360 | options.passthroughOptions.preconditionOpts.ifGenerationMatch = 0;
|
361 | }
|
362 | else if (options.skipIfExists &&
|
363 | options.passthroughOptions === undefined) {
|
364 | options.passthroughOptions = {
|
365 | preconditionOpts: {
|
366 | ifGenerationMatch: 0,
|
367 | },
|
368 | };
|
369 | }
|
370 | const limit = (0, p_limit_1.default)(options.concurrencyLimit || DEFAULT_PARALLEL_UPLOAD_LIMIT);
|
371 | const promises = [];
|
372 | let allPaths = [];
|
373 | if (!Array.isArray(filePathsOrDirectory)) {
|
374 | for await (const curPath of this.getPathsFromDirectory(filePathsOrDirectory)) {
|
375 | allPaths.push(curPath);
|
376 | }
|
377 | }
|
378 | else {
|
379 | allPaths = filePathsOrDirectory;
|
380 | }
|
381 | for (const filePath of allPaths) {
|
382 | const stat = await fs_1.promises.lstat(filePath);
|
383 | if (stat.isDirectory()) {
|
384 | continue;
|
385 | }
|
386 | const passThroughOptionsCopy = {
|
387 | ...options.passthroughOptions,
|
388 | [util_js_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.UPLOAD_MANY,
|
389 | };
|
390 | passThroughOptionsCopy.destination = options.customDestinationBuilder
|
391 | ? options.customDestinationBuilder(filePath, options)
|
392 | : filePath.split(path.sep).join(path.posix.sep);
|
393 | if (options.prefix) {
|
394 | passThroughOptionsCopy.destination = path.posix.join(...options.prefix.split(path.sep), passThroughOptionsCopy.destination);
|
395 | }
|
396 | promises.push(limit(() => this.bucket.upload(filePath, passThroughOptionsCopy)));
|
397 | }
|
398 | return Promise.all(promises);
|
399 | }
|
400 | |
401 |
|
402 |
|
403 |
|
404 |
|
405 |
|
406 |
|
407 |
|
408 |
|
409 |
|
410 | |
411 |
|
412 |
|
413 |
|
414 |
|
415 |
|
416 |
|
417 |
|
418 |
|
419 |
|
420 |
|
421 |
|
422 |
|
423 |
|
424 |
|
425 |
|
426 |
|
427 |
|
428 |
|
429 |
|
430 |
|
431 |
|
432 |
|
433 |
|
434 |
|
435 |
|
436 |
|
437 |
|
438 |
|
439 |
|
440 |
|
441 |
|
442 |
|
443 |
|
444 | async downloadManyFiles(filesOrFolder, options = {}) {
|
445 | const limit = (0, p_limit_1.default)(options.concurrencyLimit || DEFAULT_PARALLEL_DOWNLOAD_LIMIT);
|
446 | const promises = [];
|
447 | let files = [];
|
448 | if (!Array.isArray(filesOrFolder)) {
|
449 | const directoryFiles = await this.bucket.getFiles({
|
450 | prefix: filesOrFolder,
|
451 | });
|
452 | files = directoryFiles[0];
|
453 | }
|
454 | else {
|
455 | files = filesOrFolder.map(curFile => {
|
456 | if (typeof curFile === 'string') {
|
457 | return this.bucket.file(curFile);
|
458 | }
|
459 | return curFile;
|
460 | });
|
461 | }
|
462 | const stripRegexString = options.stripPrefix
|
463 | ? `^${options.stripPrefix}`
|
464 | : EMPTY_REGEX;
|
465 | const regex = new RegExp(stripRegexString, 'g');
|
466 | for (const file of files) {
|
467 | const passThroughOptionsCopy = {
|
468 | ...options.passthroughOptions,
|
469 | [util_js_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_MANY,
|
470 | };
|
471 | if (options.prefix || passThroughOptionsCopy.destination) {
|
472 | passThroughOptionsCopy.destination = path.join(options.prefix || '', passThroughOptionsCopy.destination || '', file.name);
|
473 | }
|
474 | if (options.stripPrefix) {
|
475 | passThroughOptionsCopy.destination = file.name.replace(regex, '');
|
476 | }
|
477 | promises.push(limit(async () => {
|
478 | const destination = passThroughOptionsCopy.destination;
|
479 | if (destination && destination.endsWith(path.sep)) {
|
480 | await fs_1.promises.mkdir(destination, { recursive: true });
|
481 | return Promise.resolve([
|
482 | Buffer.alloc(0),
|
483 | ]);
|
484 | }
|
485 | return file.download(passThroughOptionsCopy);
|
486 | }));
|
487 | }
|
488 | return Promise.all(promises);
|
489 | }
|
490 | |
491 |
|
492 |
|
493 |
|
494 |
|
495 |
|
496 |
|
497 |
|
498 |
|
499 | |
500 |
|
501 |
|
502 |
|
503 |
|
504 |
|
505 |
|
506 |
|
507 |
|
508 |
|
509 |
|
510 |
|
511 |
|
512 |
|
513 |
|
514 |
|
515 |
|
516 |
|
517 |
|
518 |
|
519 |
|
520 |
|
521 |
|
522 |
|
523 | async downloadFileInChunks(fileOrName, options = {}) {
|
524 | let chunkSize = options.chunkSizeBytes || DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE;
|
525 | let limit = (0, p_limit_1.default)(options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT);
|
526 | const noReturnData = Boolean(options.noReturnData);
|
527 | const promises = [];
|
528 | const file = typeof fileOrName === 'string'
|
529 | ? this.bucket.file(fileOrName)
|
530 | : fileOrName;
|
531 | const fileInfo = await file.get();
|
532 | const size = parseInt(fileInfo[0].metadata.size.toString());
|
533 |
|
534 | if (size < DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD) {
|
535 | limit = (0, p_limit_1.default)(1);
|
536 | chunkSize = size;
|
537 | }
|
538 | let start = 0;
|
539 | const filePath = options.destination || path.basename(file.name);
|
540 | const fileToWrite = await fs_1.promises.open(filePath, 'w');
|
541 | while (start < size) {
|
542 | const chunkStart = start;
|
543 | let chunkEnd = start + chunkSize - 1;
|
544 | chunkEnd = chunkEnd > size ? size : chunkEnd;
|
545 | promises.push(limit(async () => {
|
546 | const resp = await file.download({
|
547 | start: chunkStart,
|
548 | end: chunkEnd,
|
549 | [util_js_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_SHARDED,
|
550 | });
|
551 | const result = await fileToWrite.write(resp[0], 0, resp[0].length, chunkStart);
|
552 | if (noReturnData)
|
553 | return;
|
554 | return result.buffer;
|
555 | }));
|
556 | start += chunkSize;
|
557 | }
|
558 | let chunks;
|
559 | try {
|
560 | chunks = await Promise.all(promises);
|
561 | }
|
562 | finally {
|
563 | await fileToWrite.close();
|
564 | }
|
565 | if (options.validation === 'crc32c' && fileInfo[0].metadata.crc32c) {
|
566 | const downloadedCrc32C = await crc32c_js_1.CRC32C.fromFile(filePath);
|
567 | if (!downloadedCrc32C.validate(fileInfo[0].metadata.crc32c)) {
|
568 | const mismatchError = new file_js_1.RequestError(file_js_1.FileExceptionMessages.DOWNLOAD_MISMATCH);
|
569 | mismatchError.code = 'CONTENT_DOWNLOAD_MISMATCH';
|
570 | throw mismatchError;
|
571 | }
|
572 | }
|
573 | if (noReturnData)
|
574 | return;
|
575 | return [Buffer.concat(chunks, size)];
|
576 | }
|
577 | |
578 |
|
579 |
|
580 |
|
581 |
|
582 |
|
583 |
|
584 |
|
585 |
|
586 |
|
587 |
|
588 |
|
589 |
|
590 |
|
591 |
|
592 |
|
593 |
|
594 | |
595 |
|
596 |
|
597 |
|
598 |
|
599 |
|
600 |
|
601 |
|
602 |
|
603 |
|
604 |
|
605 |
|
606 |
|
607 |
|
608 |
|
609 |
|
610 |
|
611 |
|
612 |
|
613 |
|
614 |
|
615 |
|
616 |
|
617 |
|
618 |
|
619 |
|
620 |
|
621 | async uploadFileInChunks(filePath, options = {}, generator = defaultMultiPartGenerator) {
|
622 | const chunkSize = options.chunkSizeBytes || UPLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE;
|
623 | const limit = (0, p_limit_1.default)(options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT);
|
624 | const maxQueueSize = options.maxQueueSize ||
|
625 | options.concurrencyLimit ||
|
626 | DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT;
|
627 | const fileName = options.uploadName || path.basename(filePath);
|
628 | const mpuHelper = generator(this.bucket, fileName, options.uploadId, options.partsMap);
|
629 | let partNumber = 1;
|
630 | let promises = [];
|
631 | try {
|
632 | if (options.uploadId === undefined) {
|
633 | await mpuHelper.initiateUpload(options.headers);
|
634 | }
|
635 | const startOrResumptionByte = mpuHelper.partsMap.size * chunkSize;
|
636 | const readStream = (0, fs_1.createReadStream)(filePath, {
|
637 | highWaterMark: chunkSize,
|
638 | start: startOrResumptionByte,
|
639 | });
|
640 |
|
641 |
|
642 | for await (const curChunk of readStream) {
|
643 | if (promises.length >= maxQueueSize) {
|
644 | await Promise.all(promises);
|
645 | promises = [];
|
646 | }
|
647 | promises.push(limit(() => mpuHelper.uploadPart(partNumber++, curChunk, options.validation)));
|
648 | }
|
649 | await Promise.all(promises);
|
650 | return await mpuHelper.completeUpload();
|
651 | }
|
652 | catch (e) {
|
653 | if ((options.autoAbortFailure === undefined || options.autoAbortFailure) &&
|
654 | mpuHelper.uploadId) {
|
655 | try {
|
656 | await mpuHelper.abortUpload();
|
657 | return;
|
658 | }
|
659 | catch (e) {
|
660 | throw new MultiPartUploadError(e.message, mpuHelper.uploadId, mpuHelper.partsMap);
|
661 | }
|
662 | }
|
663 | throw new MultiPartUploadError(e.message, mpuHelper.uploadId, mpuHelper.partsMap);
|
664 | }
|
665 | }
|
666 | async *getPathsFromDirectory(directory) {
|
667 | const filesAndSubdirectories = await fs_1.promises.readdir(directory, {
|
668 | withFileTypes: true,
|
669 | });
|
670 | for (const curFileOrDirectory of filesAndSubdirectories) {
|
671 | const fullPath = path.join(directory, curFileOrDirectory.name);
|
672 | curFileOrDirectory.isDirectory()
|
673 | ? yield* this.getPathsFromDirectory(fullPath)
|
674 | : yield fullPath;
|
675 | }
|
676 | }
|
677 | }
|
678 | exports.TransferManager = TransferManager;
|