UNPKG

31 kBJavaScriptView Raw
1"use strict";
2/*!
3 * Copyright 2022 Google LLC. All Rights Reserved.
4 *
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
18 if (k2 === undefined) k2 = k;
19 var desc = Object.getOwnPropertyDescriptor(m, k);
20 if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
21 desc = { enumerable: true, get: function() { return m[k]; } };
22 }
23 Object.defineProperty(o, k2, desc);
24}) : (function(o, m, k, k2) {
25 if (k2 === undefined) k2 = k;
26 o[k2] = m[k];
27}));
28var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
29 Object.defineProperty(o, "default", { enumerable: true, value: v });
30}) : function(o, v) {
31 o["default"] = v;
32});
33var __importStar = (this && this.__importStar) || function (mod) {
34 if (mod && mod.__esModule) return mod;
35 var result = {};
36 if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
37 __setModuleDefault(result, mod);
38 return result;
39};
40var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
41 if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
42 if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
43 return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
44};
45var __importDefault = (this && this.__importDefault) || function (mod) {
46 return (mod && mod.__esModule) ? mod : { "default": mod };
47};
48var _XMLMultiPartUploadHelper_instances, _XMLMultiPartUploadHelper_setGoogApiClientHeaders, _XMLMultiPartUploadHelper_handleErrorResponse;
49Object.defineProperty(exports, "__esModule", { value: true });
50exports.TransferManager = exports.MultiPartUploadError = void 0;
51const file_js_1 = require("./file.js");
52const p_limit_1 = __importDefault(require("p-limit"));
53const path = __importStar(require("path"));
54const fs_1 = require("fs");
55const crc32c_js_1 = require("./crc32c.js");
56const google_auth_library_1 = require("google-auth-library");
57const fast_xml_parser_1 = require("fast-xml-parser");
58const async_retry_1 = __importDefault(require("async-retry"));
59const crypto_1 = require("crypto");
60const util_js_1 = require("./nodejs-common/util.js");
61const util_js_2 = require("./util.js");
62// eslint-disable-next-line @typescript-eslint/ban-ts-comment
63// @ts-ignore
64const package_json_helper_cjs_1 = require("./package-json-helper.cjs");
65const packageJson = (0, package_json_helper_cjs_1.getPackageJSON)();
66/**
67 * Default number of concurrently executing promises to use when calling uploadManyFiles.
68 *
69 */
70const DEFAULT_PARALLEL_UPLOAD_LIMIT = 5;
71/**
72 * Default number of concurrently executing promises to use when calling downloadManyFiles.
73 *
74 */
75const DEFAULT_PARALLEL_DOWNLOAD_LIMIT = 5;
76/**
77 * Default number of concurrently executing promises to use when calling downloadFileInChunks.
78 *
79 */
80const DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT = 5;
81/**
82 * The minimum size threshold in bytes at which to apply a chunked download strategy when calling downloadFileInChunks.
83 *
84 */
85const DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD = 32 * 1024 * 1024;
86/**
87 * The chunk size in bytes to use when calling downloadFileInChunks.
88 *
89 */
90const DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024;
91/**
92 * The chunk size in bytes to use when calling uploadFileInChunks.
93 *
94 */
95const UPLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE = 32 * 1024 * 1024;
96/**
97 * Default number of concurrently executing promises to use when calling uploadFileInChunks.
98 *
99 */
100const DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT = 5;
101const EMPTY_REGEX = '(?:)';
102/**
103 * The `gccl-gcs-cmd` value for the `X-Goog-API-Client` header.
104 * Example: `gccl-gcs-cmd/tm.upload_many`
105 *
106 * @see {@link GCCL_GCS_CMD}.
107 * @see {@link GCCL_GCS_CMD_KEY}.
108 */
109const GCCL_GCS_CMD_FEATURE = {
110 UPLOAD_MANY: 'tm.upload_many',
111 DOWNLOAD_MANY: 'tm.download_many',
112 UPLOAD_SHARDED: 'tm.upload_sharded',
113 DOWNLOAD_SHARDED: 'tm.download_sharded',
114};
115const defaultMultiPartGenerator = (bucket, fileName, uploadId, partsMap) => {
116 return new XMLMultiPartUploadHelper(bucket, fileName, uploadId, partsMap);
117};
118class MultiPartUploadError extends Error {
119 constructor(message, uploadId, partsMap) {
120 super(message);
121 this.uploadId = uploadId;
122 this.partsMap = partsMap;
123 }
124}
125exports.MultiPartUploadError = MultiPartUploadError;
126/**
127 * Class representing an implementation of MPU in the XML API. This class is not meant for public usage.
128 *
129 * @private
130 *
131 */
132class XMLMultiPartUploadHelper {
133 constructor(bucket, fileName, uploadId, partsMap) {
134 _XMLMultiPartUploadHelper_instances.add(this);
135 this.authClient = bucket.storage.authClient || new google_auth_library_1.GoogleAuth();
136 this.uploadId = uploadId || '';
137 this.bucket = bucket;
138 this.fileName = fileName;
139 this.baseUrl = `https://${bucket.name}.${new URL(this.bucket.storage.apiEndpoint).hostname}/${fileName}`;
140 this.xmlBuilder = new fast_xml_parser_1.XMLBuilder({ arrayNodeName: 'Part' });
141 this.xmlParser = new fast_xml_parser_1.XMLParser();
142 this.partsMap = partsMap || new Map();
143 this.retryOptions = {
144 retries: this.bucket.storage.retryOptions.maxRetries,
145 factor: this.bucket.storage.retryOptions.retryDelayMultiplier,
146 maxTimeout: this.bucket.storage.retryOptions.maxRetryDelay * 1000,
147 maxRetryTime: this.bucket.storage.retryOptions.totalTimeout * 1000,
148 };
149 }
150 /**
151 * Initiates a multipart upload (MPU) to the XML API and stores the resultant upload id.
152 *
153 * @returns {Promise<void>}
154 */
155 async initiateUpload(headers = {}) {
156 const url = `${this.baseUrl}?uploads`;
157 return (0, async_retry_1.default)(async (bail) => {
158 try {
159 const res = await this.authClient.request({
160 headers: __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this, headers),
161 method: 'POST',
162 url,
163 });
164 if (res.data && res.data.error) {
165 throw res.data.error;
166 }
167 const parsedXML = this.xmlParser.parse(res.data);
168 this.uploadId = parsedXML.InitiateMultipartUploadResult.UploadId;
169 }
170 catch (e) {
171 __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
172 }
173 }, this.retryOptions);
174 }
175 /**
176 * Uploads the provided chunk of data to the XML API using the previously created upload id.
177 *
178 * @param {number} partNumber the sequence number of this chunk.
179 * @param {Buffer} chunk the chunk of data to be uploaded.
180 * @param {string | false} validation whether or not to include the md5 hash in the headers to cause the server
181 * to validate the chunk was not corrupted.
182 * @returns {Promise<void>}
183 */
184 async uploadPart(partNumber, chunk, validation) {
185 const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`;
186 let headers = __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this);
187 if (validation === 'md5') {
188 const hash = (0, crypto_1.createHash)('md5').update(chunk).digest('base64');
189 headers = {
190 'Content-MD5': hash,
191 };
192 }
193 return (0, async_retry_1.default)(async (bail) => {
194 try {
195 const res = await this.authClient.request({
196 url,
197 method: 'PUT',
198 body: chunk,
199 headers,
200 });
201 if (res.data && res.data.error) {
202 throw res.data.error;
203 }
204 this.partsMap.set(partNumber, res.headers['etag']);
205 }
206 catch (e) {
207 __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
208 }
209 }, this.retryOptions);
210 }
211 /**
212 * Sends the final request of the MPU to tell GCS the upload is now complete.
213 *
214 * @returns {Promise<void>}
215 */
216 async completeUpload() {
217 const url = `${this.baseUrl}?uploadId=${this.uploadId}`;
218 const sortedMap = new Map([...this.partsMap.entries()].sort((a, b) => a[0] - b[0]));
219 const parts = [];
220 for (const entry of sortedMap.entries()) {
221 parts.push({ PartNumber: entry[0], ETag: entry[1] });
222 }
223 const body = `<CompleteMultipartUpload>${this.xmlBuilder.build(parts)}</CompleteMultipartUpload>`;
224 return (0, async_retry_1.default)(async (bail) => {
225 try {
226 const res = await this.authClient.request({
227 headers: __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this),
228 url,
229 method: 'POST',
230 body,
231 });
232 if (res.data && res.data.error) {
233 throw res.data.error;
234 }
235 return res;
236 }
237 catch (e) {
238 __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
239 return;
240 }
241 }, this.retryOptions);
242 }
243 /**
244 * Aborts an multipart upload that is in progress. Once aborted, any parts in the process of being uploaded fail,
245 * and future requests using the upload ID fail.
246 *
247 * @returns {Promise<void>}
248 */
249 async abortUpload() {
250 const url = `${this.baseUrl}?uploadId=${this.uploadId}`;
251 return (0, async_retry_1.default)(async (bail) => {
252 try {
253 const res = await this.authClient.request({
254 url,
255 method: 'DELETE',
256 });
257 if (res.data && res.data.error) {
258 throw res.data.error;
259 }
260 }
261 catch (e) {
262 __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
263 return;
264 }
265 }, this.retryOptions);
266 }
267}
268_XMLMultiPartUploadHelper_instances = new WeakSet(), _XMLMultiPartUploadHelper_setGoogApiClientHeaders = function _XMLMultiPartUploadHelper_setGoogApiClientHeaders(headers = {}) {
269 let headerFound = false;
270 let userAgentFound = false;
271 for (const [key, value] of Object.entries(headers)) {
272 if (key.toLocaleLowerCase().trim() === 'x-goog-api-client') {
273 headerFound = true;
274 // Prepend command feature to value, if not already there
275 if (!value.includes(GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED)) {
276 headers[key] =
277 `${value} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`;
278 }
279 }
280 else if (key.toLocaleLowerCase().trim() === 'user-agent') {
281 userAgentFound = true;
282 }
283 }
284 // If the header isn't present, add it
285 if (!headerFound) {
286 headers['x-goog-api-client'] = `${(0, util_js_2.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`;
287 }
288 // If the User-Agent isn't present, add it
289 if (!userAgentFound) {
290 headers['User-Agent'] = (0, util_js_2.getUserAgentString)();
291 }
292 return headers;
293}, _XMLMultiPartUploadHelper_handleErrorResponse = function _XMLMultiPartUploadHelper_handleErrorResponse(err, bail) {
294 if (this.bucket.storage.retryOptions.autoRetry &&
295 this.bucket.storage.retryOptions.retryableErrorFn(err)) {
296 throw err;
297 }
298 else {
299 bail(err);
300 }
301};
302/**
303 * Create a TransferManager object to perform parallel transfer operations on a Cloud Storage bucket.
304 *
305 * @class
306 * @hideconstructor
307 *
308 * @param {Bucket} bucket A {@link Bucket} instance
309 *
310 */
311class TransferManager {
312 constructor(bucket) {
313 this.bucket = bucket;
314 }
315 /**
316 * @typedef {object} UploadManyFilesOptions
317 * @property {number} [concurrencyLimit] The number of concurrently executing promises
318 * to use when uploading the files.
319 * @property {Function} [customDestinationBuilder] A fuction that will take the current path of a local file
320 * and return a string representing a custom path to be used to upload the file to GCS.
321 * @property {boolean} [skipIfExists] Do not upload the file if it already exists in
322 * the bucket. This will set the precondition ifGenerationMatch = 0.
323 * @property {string} [prefix] A prefix to append to all of the uploaded files.
324 * @property {object} [passthroughOptions] {@link UploadOptions} Options to be passed through
325 * to each individual upload operation.
326 *
327 */
328 /**
329 * Upload multiple files in parallel to the bucket. This is a convenience method
330 * that utilizes {@link Bucket#upload} to perform the upload.
331 *
332 * @param {array | string} [filePathsOrDirectory] An array of fully qualified paths to the files or a directory name.
333 * If a directory name is provided, the directory will be recursively walked and all files will be added to the upload list.
334 * to be uploaded to the bucket
335 * @param {UploadManyFilesOptions} [options] Configuration options.
336 * @returns {Promise<UploadResponse[]>}
337 *
338 * @example
339 * ```
340 * const {Storage} = require('@google-cloud/storage');
341 * const storage = new Storage();
342 * const bucket = storage.bucket('my-bucket');
343 * const transferManager = new TransferManager(bucket);
344 *
345 * //-
346 * // Upload multiple files in parallel.
347 * //-
348 * const response = await transferManager.uploadManyFiles(['/local/path/file1.txt, 'local/path/file2.txt']);
349 * // Your bucket now contains:
350 * // - "local/path/file1.txt" (with the contents of '/local/path/file1.txt')
351 * // - "local/path/file2.txt" (with the contents of '/local/path/file2.txt')
352 * const response = await transferManager.uploadManyFiles('/local/directory');
353 * // Your bucket will now contain all files contained in '/local/directory' maintaining the subdirectory structure.
354 * ```
355 *
356 */
357 async uploadManyFiles(filePathsOrDirectory, options = {}) {
358 var _a;
359 if (options.skipIfExists && ((_a = options.passthroughOptions) === null || _a === void 0 ? void 0 : _a.preconditionOpts)) {
360 options.passthroughOptions.preconditionOpts.ifGenerationMatch = 0;
361 }
362 else if (options.skipIfExists &&
363 options.passthroughOptions === undefined) {
364 options.passthroughOptions = {
365 preconditionOpts: {
366 ifGenerationMatch: 0,
367 },
368 };
369 }
370 const limit = (0, p_limit_1.default)(options.concurrencyLimit || DEFAULT_PARALLEL_UPLOAD_LIMIT);
371 const promises = [];
372 let allPaths = [];
373 if (!Array.isArray(filePathsOrDirectory)) {
374 for await (const curPath of this.getPathsFromDirectory(filePathsOrDirectory)) {
375 allPaths.push(curPath);
376 }
377 }
378 else {
379 allPaths = filePathsOrDirectory;
380 }
381 for (const filePath of allPaths) {
382 const stat = await fs_1.promises.lstat(filePath);
383 if (stat.isDirectory()) {
384 continue;
385 }
386 const passThroughOptionsCopy = {
387 ...options.passthroughOptions,
388 [util_js_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.UPLOAD_MANY,
389 };
390 passThroughOptionsCopy.destination = options.customDestinationBuilder
391 ? options.customDestinationBuilder(filePath, options)
392 : filePath.split(path.sep).join(path.posix.sep);
393 if (options.prefix) {
394 passThroughOptionsCopy.destination = path.posix.join(...options.prefix.split(path.sep), passThroughOptionsCopy.destination);
395 }
396 promises.push(limit(() => this.bucket.upload(filePath, passThroughOptionsCopy)));
397 }
398 return Promise.all(promises);
399 }
400 /**
401 * @typedef {object} DownloadManyFilesOptions
402 * @property {number} [concurrencyLimit] The number of concurrently executing promises
403 * to use when downloading the files.
404 * @property {string} [prefix] A prefix to append to all of the downloaded files.
405 * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files.
406 * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through
407 * to each individual download operation.
408 *
409 */
410 /**
411 * Download multiple files in parallel to the local filesystem. This is a convenience method
412 * that utilizes {@link File#download} to perform the download.
413 *
414 * @param {array | string} [filesOrFolder] An array of file name strings or file objects to be downloaded. If
415 * a string is provided this will be treated as a GCS prefix and all files with that prefix will be downloaded.
416 * @param {DownloadManyFilesOptions} [options] Configuration options. Setting options.prefix or options.stripPrefix
417 * or options.passthroughOptions.destination will cause the downloaded files to be written to the file system
418 * instead of being returned as a buffer.
419 * @returns {Promise<DownloadResponse[]>}
420 *
421 * @example
422 * ```
423 * const {Storage} = require('@google-cloud/storage');
424 * const storage = new Storage();
425 * const bucket = storage.bucket('my-bucket');
426 * const transferManager = new TransferManager(bucket);
427 *
428 * //-
429 * // Download multiple files in parallel.
430 * //-
431 * const response = await transferManager.downloadManyFiles(['file1.txt', 'file2.txt']);
432 * // The following files have been downloaded:
433 * // - "file1.txt" (with the contents from my-bucket.file1.txt)
434 * // - "file2.txt" (with the contents from my-bucket.file2.txt)
435 * const response = await transferManager.downloadManyFiles([bucket.File('file1.txt'), bucket.File('file2.txt')]);
436 * // The following files have been downloaded:
437 * // - "file1.txt" (with the contents from my-bucket.file1.txt)
438 * // - "file2.txt" (with the contents from my-bucket.file2.txt)
439 * const response = await transferManager.downloadManyFiles('test-folder');
440 * // All files with GCS prefix of 'test-folder' have been downloaded.
441 * ```
442 *
443 */
444 async downloadManyFiles(filesOrFolder, options = {}) {
445 const limit = (0, p_limit_1.default)(options.concurrencyLimit || DEFAULT_PARALLEL_DOWNLOAD_LIMIT);
446 const promises = [];
447 let files = [];
448 if (!Array.isArray(filesOrFolder)) {
449 const directoryFiles = await this.bucket.getFiles({
450 prefix: filesOrFolder,
451 });
452 files = directoryFiles[0];
453 }
454 else {
455 files = filesOrFolder.map(curFile => {
456 if (typeof curFile === 'string') {
457 return this.bucket.file(curFile);
458 }
459 return curFile;
460 });
461 }
462 const stripRegexString = options.stripPrefix
463 ? `^${options.stripPrefix}`
464 : EMPTY_REGEX;
465 const regex = new RegExp(stripRegexString, 'g');
466 for (const file of files) {
467 const passThroughOptionsCopy = {
468 ...options.passthroughOptions,
469 [util_js_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_MANY,
470 };
471 if (options.prefix || passThroughOptionsCopy.destination) {
472 passThroughOptionsCopy.destination = path.join(options.prefix || '', passThroughOptionsCopy.destination || '', file.name);
473 }
474 if (options.stripPrefix) {
475 passThroughOptionsCopy.destination = file.name.replace(regex, '');
476 }
477 promises.push(limit(async () => {
478 const destination = passThroughOptionsCopy.destination;
479 if (destination && destination.endsWith(path.sep)) {
480 await fs_1.promises.mkdir(destination, { recursive: true });
481 return Promise.resolve([
482 Buffer.alloc(0),
483 ]);
484 }
485 return file.download(passThroughOptionsCopy);
486 }));
487 }
488 return Promise.all(promises);
489 }
490 /**
491 * @typedef {object} DownloadFileInChunksOptions
492 * @property {number} [concurrencyLimit] The number of concurrently executing promises
493 * to use when downloading the file.
494 * @property {number} [chunkSizeBytes] The size in bytes of each chunk to be downloaded.
495 * @property {string | boolean} [validation] Whether or not to perform a CRC32C validation check when download is complete.
496 * @property {boolean} [noReturnData] Whether or not to return the downloaded data. A `true` value here would be useful for files with a size that will not fit into memory.
497 *
498 */
499 /**
500 * Download a large file in chunks utilizing parallel download operations. This is a convenience method
501 * that utilizes {@link File#download} to perform the download.
502 *
503 * @param {File | string} fileOrName {@link File} to download.
504 * @param {DownloadFileInChunksOptions} [options] Configuration options.
505 * @returns {Promise<void | DownloadResponse>}
506 *
507 * @example
508 * ```
509 * const {Storage} = require('@google-cloud/storage');
510 * const storage = new Storage();
511 * const bucket = storage.bucket('my-bucket');
512 * const transferManager = new TransferManager(bucket);
513 *
514 * //-
515 * // Download a large file in chunks utilizing parallel operations.
516 * //-
517 * const response = await transferManager.downloadFileInChunks(bucket.file('large-file.txt');
518 * // Your local directory now contains:
519 * // - "large-file.txt" (with the contents from my-bucket.large-file.txt)
520 * ```
521 *
522 */
523 async downloadFileInChunks(fileOrName, options = {}) {
524 let chunkSize = options.chunkSizeBytes || DOWNLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE;
525 let limit = (0, p_limit_1.default)(options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_DOWNLOAD_LIMIT);
526 const noReturnData = Boolean(options.noReturnData);
527 const promises = [];
528 const file = typeof fileOrName === 'string'
529 ? this.bucket.file(fileOrName)
530 : fileOrName;
531 const fileInfo = await file.get();
532 const size = parseInt(fileInfo[0].metadata.size.toString());
533 // If the file size does not meet the threshold download it as a single chunk.
534 if (size < DOWNLOAD_IN_CHUNKS_FILE_SIZE_THRESHOLD) {
535 limit = (0, p_limit_1.default)(1);
536 chunkSize = size;
537 }
538 let start = 0;
539 const filePath = options.destination || path.basename(file.name);
540 const fileToWrite = await fs_1.promises.open(filePath, 'w');
541 while (start < size) {
542 const chunkStart = start;
543 let chunkEnd = start + chunkSize - 1;
544 chunkEnd = chunkEnd > size ? size : chunkEnd;
545 promises.push(limit(async () => {
546 const resp = await file.download({
547 start: chunkStart,
548 end: chunkEnd,
549 [util_js_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_SHARDED,
550 });
551 const result = await fileToWrite.write(resp[0], 0, resp[0].length, chunkStart);
552 if (noReturnData)
553 return;
554 return result.buffer;
555 }));
556 start += chunkSize;
557 }
558 let chunks;
559 try {
560 chunks = await Promise.all(promises);
561 }
562 finally {
563 await fileToWrite.close();
564 }
565 if (options.validation === 'crc32c' && fileInfo[0].metadata.crc32c) {
566 const downloadedCrc32C = await crc32c_js_1.CRC32C.fromFile(filePath);
567 if (!downloadedCrc32C.validate(fileInfo[0].metadata.crc32c)) {
568 const mismatchError = new file_js_1.RequestError(file_js_1.FileExceptionMessages.DOWNLOAD_MISMATCH);
569 mismatchError.code = 'CONTENT_DOWNLOAD_MISMATCH';
570 throw mismatchError;
571 }
572 }
573 if (noReturnData)
574 return;
575 return [Buffer.concat(chunks, size)];
576 }
577 /**
578 * @typedef {object} UploadFileInChunksOptions
579 * @property {number} [concurrencyLimit] The number of concurrently executing promises
580 * to use when uploading the file.
581 * @property {number} [chunkSizeBytes] The size in bytes of each chunk to be uploaded.
582 * @property {string} [uploadName] Name of the file when saving to GCS. If ommitted the name is taken from the file path.
583 * @property {number} [maxQueueSize] The number of chunks to be uploaded to hold in memory concurrently. If not specified
584 * defaults to the specified concurrency limit.
585 * @property {string} [uploadId] If specified attempts to resume a previous upload.
586 * @property {Map} [partsMap] If specified alongside uploadId, attempts to resume a previous upload from the last chunk
587 * specified in partsMap
588 * @property {object} [headers] headers to be sent when initiating the multipart upload.
589 * See {@link https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers| Request Headers: Initiate a Multipart Upload}
590 * @property {boolean} [autoAbortFailure] boolean to indicate if an in progress upload session will be automatically aborted upon failure. If not set,
591 * failures will be automatically aborted.
592 *
593 */
594 /**
595 * Upload a large file in chunks utilizing parallel upload opertions. If the upload fails, an uploadId and
596 * map containing all the successfully uploaded parts will be returned to the caller. These arguments can be used to
597 * resume the upload.
598 *
599 * @param {string} [filePath] The path of the file to be uploaded
600 * @param {UploadFileInChunksOptions} [options] Configuration options.
601 * @param {MultiPartHelperGenerator} [generator] A function that will return a type that implements the MPU interface. Most users will not need to use this.
602 * @returns {Promise<void>} If successful a promise resolving to void, otherwise a error containing the message, uploadid, and parts map.
603 *
604 * @example
605 * ```
606 * const {Storage} = require('@google-cloud/storage');
607 * const storage = new Storage();
608 * const bucket = storage.bucket('my-bucket');
609 * const transferManager = new TransferManager(bucket);
610 *
611 * //-
612 * // Upload a large file in chunks utilizing parallel operations.
613 * //-
614 * const response = await transferManager.uploadFileInChunks('large-file.txt');
615 * // Your bucket now contains:
616 * // - "large-file.txt"
617 * ```
618 *
619 *
620 */
621 async uploadFileInChunks(filePath, options = {}, generator = defaultMultiPartGenerator) {
622 const chunkSize = options.chunkSizeBytes || UPLOAD_IN_CHUNKS_DEFAULT_CHUNK_SIZE;
623 const limit = (0, p_limit_1.default)(options.concurrencyLimit || DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT);
624 const maxQueueSize = options.maxQueueSize ||
625 options.concurrencyLimit ||
626 DEFAULT_PARALLEL_CHUNKED_UPLOAD_LIMIT;
627 const fileName = options.uploadName || path.basename(filePath);
628 const mpuHelper = generator(this.bucket, fileName, options.uploadId, options.partsMap);
629 let partNumber = 1;
630 let promises = [];
631 try {
632 if (options.uploadId === undefined) {
633 await mpuHelper.initiateUpload(options.headers);
634 }
635 const startOrResumptionByte = mpuHelper.partsMap.size * chunkSize;
636 const readStream = (0, fs_1.createReadStream)(filePath, {
637 highWaterMark: chunkSize,
638 start: startOrResumptionByte,
639 });
640 // p-limit only limits the number of running promises. We do not want to hold an entire
641 // large file in memory at once so promises acts a queue that will hold only maxQueueSize in memory.
642 for await (const curChunk of readStream) {
643 if (promises.length >= maxQueueSize) {
644 await Promise.all(promises);
645 promises = [];
646 }
647 promises.push(limit(() => mpuHelper.uploadPart(partNumber++, curChunk, options.validation)));
648 }
649 await Promise.all(promises);
650 return await mpuHelper.completeUpload();
651 }
652 catch (e) {
653 if ((options.autoAbortFailure === undefined || options.autoAbortFailure) &&
654 mpuHelper.uploadId) {
655 try {
656 await mpuHelper.abortUpload();
657 return;
658 }
659 catch (e) {
660 throw new MultiPartUploadError(e.message, mpuHelper.uploadId, mpuHelper.partsMap);
661 }
662 }
663 throw new MultiPartUploadError(e.message, mpuHelper.uploadId, mpuHelper.partsMap);
664 }
665 }
666 async *getPathsFromDirectory(directory) {
667 const filesAndSubdirectories = await fs_1.promises.readdir(directory, {
668 withFileTypes: true,
669 });
670 for (const curFileOrDirectory of filesAndSubdirectories) {
671 const fullPath = path.join(directory, curFileOrDirectory.name);
672 curFileOrDirectory.isDirectory()
673 ? yield* this.getPathsFromDirectory(fullPath)
674 : yield fullPath;
675 }
676 }
677}
678exports.TransferManager = TransferManager;