| 1 | {"version":3,"file":"uploader.js","sourceRoot":"","sources":["../src/uploader.ts"],"names":[],"mappings":";;;;;;;AAEA,AAAO,AAAe,AAAM,AAAc;;;;;;;;;;AAF1C,AAAO,AAAE,AAAM,AAAI,AAAS,AAAM,AAAM,AAAS;;;;;;;;;;AAGjD,AAAO,AAAE,AAAQ,AAAE,AAAM,AAAc;;;;;;;;;;AACvC,AAAO,AAAE,AAAY,AAAE,AAAM,AAAQ;;;;;;;;;;AACrC,AAAO,AAAE,AAAgB,AAAE,AAAM,AAAY;;;;;;;;;;AAC7C,AAAO,AAAE,AAAI,AAAE,AAAM,AAAI;;;;;;;;;;AACzB,AAAO,AAAE,AAAU,AAAE,AAAM,AAAQ;;;;;;;;;;;;AAEnC,MAAM,AAAmB,sBAAG,AAAC,IAAG,AAAI,OAAG,AAAI,OAAG,AAAI;AAClD,MAAM,AAAmB,sBAAG,AAAK;AACjC,MAAM,AAAkB,qBAAG,AAAC,IAAG,AAAI,OAAG,AAAI;AAC1C,MAAM,AAAgB,mBAAG,AAAE,KAAG,AAAI,OAAG,AAAI;;AAEzC,AAAS,iBAAC,AAAqB,sBAAC,AAAO,QAAC,AAAc,AAAC,AAAC,AAExD,AAAM;;MAAgB,iBAAQ,AAAY;AAaxC,cAA6B,AAAM,IAAmB,AAAuC,WAAmB,AAAiB,WAAW,AAAqB,eAAmB,AAAsC;AACxN,AAAK,AAAE;AADoB,SAAE,KAAF,AAAE,AAAI;AAAmB,SAAS,YAAT,AAAS,AAA8B;AAAmB,SAAS,YAAT,AAAS,AAAQ;AAAW,SAAa,gBAAb,AAAa,AAAQ;AAAmB,SAAW,cAAX,AAAW,AAA2B;AAZ1N,AAAgB;;AAChB,SAAM,SAAG,AAAC;AAEF,SAAS,YAAG,AAAK;AAYvB,AAAI,SAAC,AAAY,eAAG,AAAC;AACrB,AAAI,SAAC,AAAY,eAAG,AAAI;AAExB,AAAI,SAAC,AAAwB,AAAG,2BAAC,AAAE,KAAG,AAAI,OAAG,AAAI,AAAC;AAClD,AAAI,SAAC,AAAmB,sBAAG,AAAgB;AAC3C,AAAI,SAAC,AAA0B,AAAG,6BAAC,AAAE,KAAG,AAAI,OAAG,AAAI,AAAC;AACpD,AAAI,SAAC,AAAqB,wBAAG,AAAgB;;AAE7C,QAAI,AAAI,KAAC,AAAwB,2BAAG,AAAkB,oBAAE;AACtD,YAAM,IAAI,AAAK,MAAC,AAA0C,AAAC;AAC5D;;AACD,QAAI,AAAI,KAAC,AAAwB,2BAAG,AAAmB,qBAAE;AACvD,YAAM,IAAI,AAAK,MAAC,AAA0C,AAAC;AAC5D;;AACD,QAAI,AAAI,KAAC,AAAmB,sBAAG,AAAkB,oBAAE;AACjD,YAAM,IAAI,AAAK,MAAC,AAAqC,AAAC;AACvD;;AACD,QAAI,AAAI,KAAC,AAAmB,sBAAG,AAAmB,qBAAE;AAClD,YAAM,IAAI,AAAK,MAAC,AAAqC,AAAC;AACvD,AACH;AAAC;;AAEK,AAAM,QAAZ,AAAK;;;;AACH,YAAM,AAAW,cAAG,AAAI,MAAC,AAAW;;AACpC,UAAI,AAAW,eAAI,AAAI,MAAE;AACvB,cAAM,AAAI,OAAG,AAAU,0BAAC,AAAK,AAAC;AAC9B,AAAI,aAAC,AAAM,OAAC,AAAW,AAAC;AACxB,cAAM,AAAG,MAAG,AAAI,KAAC,AAAM,OAAC,AAAQ,AAAC;AACjC,cAAM,AAAI,MAAC,AAAU,WAAC,AAAG,AAAE,MAAC,AAAI,MAAC,AAAS,UAAC,AAAG,AAAC,AAAC;AAChD,AAAM;AACP;;AAED,UAAI,AAAI,MAAC,AAAa,gBAAG,AAAI,MAAC,AAAwB,0BAAE;AACtD,cAAM,AAAG,MAAG,MAAM,AAAQ,6BAAC,AAAI,MAAC,AAAS,WAAE,AAAK,AAAC;AACjD,cAAM,AAAI,MAAC,AAAU,WAAC,AAAG,AAAE,MAAC,AAAI,MAAC,AAAS,UAAC,AAAG,AAAC,AAAC;AAChD,AAAM;AACP;;AAED,UAAI,AAAmB,sBAAG,AAAI,MAAC,AAAmB;;AAClD,UAAI,AAAI,KAAC,AAAI,KAAC,AAAI,MAAC,AAAa,gBAAG,AAAmB,AAAC,uBAAG,AAAmB,qBAAE;AAC7E,AAAmB,8BAAG,AAA4B,6BAAC,AAAI,MAAC,AAAa,AAAC;AACvE;;AAED,UAAI,AAAmB,sBAAG,AAAmB,qBAAE;AAC7C,cAAM,IAAI,AAAK,AAAC,gDAA0C,AAAI,MAAC,AAAS,SAAE,AAAC;AAC5E;;AAED,YAAM,AAAI,OAAG,MAAM,AAAI,MAAC,AAAU,WAAC,AAAG,AAAE,MAAC,AAAI,MAAC,AAAE,GAAC,AAAqB,sBAAC,AAAI,MAAC,AAAS,AAAC,WAAC,AAAO,AAAE,AAAC;AACjG,YAAM,AAAI,MAAC,AAAe,gBAAC,AAAI,KAAC,AAAS,UAAE,AAAmB,AAAC,AACjE;;AAAC;;AAED,AAAK;AACH,AAAI,SAAC,AAAS,YAAG,AAAI,AACvB;AAAC;;AAEO,AAAS,YAAC,AAAW;AAC3B,AAAI,SAAC,AAAM,SAAG,AAAC;AACf,gBAAW,AAAe,wBAAM,CAAC,AAAO,SAAE,AAAM,AAAE,AAAE;AAClD,AAAI,WAAC,AAAE,GAAC,AAAS;AACf,AAAI,cAAE,AAAI,KAAC,AAAW,eAAI,AAAgB,kCAAC,AAAI,KAAC,AAAS,AAAC;AAC1D,AAAU,oBAAE,AAAG;SACZ,AAAI,KAAC,AAAS,AACjB,YACC,AAAE,GAAC,AAAoB,sBAAE,AAAQ,AAAC,AAAE;AACnC,AAAI,aAAC,AAAM,SAAG,AAAQ,SAAC,AAAM;AAC7B,AAAI,aAAC,AAAI,KAAC,AAAU,AAAC,AACvB;AAAC,AAAC,SACD,AAAI,KAAC,CAAC,AAAK,OAAE,AAAI,AAAE,AAAE;AACpB,YAAI,AAAK,SAAI,AAAI,MAAE;AACjB,AAAO,kBAAC,AAAI,AAAC;AACd,eACI;AACH,AAAM,iBAAC,AAAK,AAAC;AACd,AACH;AAAC,AAAC,AACN;AAAC,AAAC,AACJ,KAnBS;AAmBR;;AAEa,AAAe,iBAArB,AAAK,CAAiB,AAAgB,UAAE,AAA2B;;;;AACzE,UAAI,AAAM,SAAG,AAAC;AACd,UAAI,AAAc,iBAAG,AAAC;AAEtB,YAAM,AAAM,SAAe,AAAE;AAE7B,YAAM,AAAK,QAAgB,AAAE;;AAC7B,aAAO,AAAM,SAAG,AAAI,OAAC,AAAa,eAAE;AAClC,cAAM,AAAK,QAAG,AAAM;AACpB,YAAI,AAAG,MAAG,AAAM,SAAG,AAAmB;;AACtC,YAAI,AAAG,MAAG,AAAI,OAAC,AAAa,eAAE;AAC5B,AAAG,gBAAG,AAAI,OAAC,AAAa;AACzB;;AACD,AAAM,iBAAG,AAAG;AACZ,cAAM,AAAI;AACR,AAAU,sBAAE,AAAc,AAAE,AAC7B;AAFY;AAGb,AAAM,eAAC,AAAI,KAAC,AAAI,AAAC;AACjB,AAAK,cAAC,AAAI;AAAE,AAAK;AAAE,AAAG;AAAE,AAAI;AAAE,AAAG,eAAE,AAAE,AAAC,AAAC;AAA5B;AACZ;;AAED,mCAAsB,AAAG,IAAC,AAAK;AAAzB,AAAe,iDAAY,AAAK,WAAC,AAAE,AAAC,AAAE;AAC1C,AAA8C;AAC9C,AAAE,aAAC,AAAG,MAAG,mCAAe,AAAI,OAAC,AAAS,WAAE,AAAK,OAAE,AAAQ;AAAG,AAAK,mBAAE,AAAE,GAAC,AAAK;AAAE,AAAG,iBAAE,AAAE,GAAC,AAAG,MAAG,AAAC,AAAC,AAAC,AAC9F;AAD2D,WAA1C,AAAQ;AACxB;;;;;;AAAG,AAAW,qBAAE,AAAI,AAAE,kBAAC,AAAM,AAAC,AAAC;AAA7B;AAEH,mCAAsB,AAAG,IAAC,AAAK,OAAE,AAAE,AAAC,AAAE,MAAC,AAAI,OAAC,AAAc,eAAC,AAAE,IAAE,AAAQ,AAAC;AAAG,AAAW,qBAAE,AAAC,AAAC,AAAC;AAAjB,OAApE,AAAe;AACrB,aAAO,aAAW,AAAU,WAAC,AAAG,AAAE,aAAM,AAAE,GAAC,AAAuB;AAC9D,AAAM,gBAAE,AAAI,OAAC,AAAS,UAAC,AAAM;AAC7B,AAAG,aAAE,AAAI,OAAC,AAAS,UAAC,AAAG;AACvB,AAAQ,kBAAE,AAAQ;AAClB,AAAe;AACb,AAAK,iBAAE,AAAM,AACd,AACF,AAAC;AAHiB;AAJ8C,OAAhC,AAAI,EAOlC,AAAO,AAAE,AACb,AACH,SATe,AAAI;;AASlB;;AAEO,AAAc,iBAAC,AAAU,MAAE,AAAgB;AACjD,UAAM,AAAa,gBAAG,AAAI,KAAC,AAAG,MAAG,AAAI,KAAC,AAAK;AAC3C,gBAAY,AAAU,WAAC,AAAG,AAAE;AAC1B,UAAI,AAAU,aAAG,AAAC;AAClB,kBAAW,AAAe,wBAAC,CAAC,AAAO,SAAE,AAAM,AAAE,AAAE;AAC7C,AAAI,aAAC,AAAE,GAAC,AAAU;AAChB,AAAa,yBAAE,AAAa;AAC5B,AAAU,sBAAE,AAAI,KAAC,AAAI,KAAC,AAAU;AAChC,AAAQ,oBAAE,AAAQ;AAClB,AAAI,kDAAmB,AAAI,KAAC,AAAS;AAAG,AAAK,mBAAE,AAAI,KAAC,AAAK;AAAE,AAAG,iBAAE,AAAI,KAAC,AAAG,MAAG,AAAC,AAAC,AAAC;AAAvC,WAAjC,AAAgB;AACtB,AAAM,kBAAE,AAAI,KAAC,AAAS,UAAC,AAAM;AAC7B,AAAG,eAAE,AAAI,KAAC,AAAS,UAAC,AAAG;AACvB,AAAU,sBAAE,AAAI,KAAC,AAAG,AACrB,AAAC;AARiB,WAShB,AAAE,GAAC,AAAoB,sBAAE,AAAQ,AAAC,AAAE;AACnC,AAAU,uBAAG,AAAQ,SAAC,AAAM;AAC5B,AAAI,eAAC,AAAM,UAAI,AAAQ,SAAC,AAAM;AAC9B,AAAI,eAAC,AAAI,KAAC,AAAU,AAAC,AACvB;AAAC,AAAC,WACD,AAAI,KAAC,CAAC,AAAK,OAAE,AAAI,AAAE,AAAE;AACpB,cAAI,AAAK,SAAI,AAAI,MAAE;AACjB,AAAI,iBAAC,AAAI,KAAC,AAAI,OAAG,AAAI,KAAC,AAAI;AAC1B,AAAO,oBAAC,AAAI,AAAC;AACd,iBACI;AACH,AAAI,iBAAC,AAAM,UAAI,AAAU;AACzB,AAAM,mBAAC,AAAK,AAAC;AACd,AACH;AAAC,AAAC,AACN;AAAC,AAAC,AACJ,OA1BS;AA0BR,AAAC,AACJ,KA7BS,AAAI;AA6BZ;;AAEa,AAAU,YAAhB,AAAK,CAAe,AAAsB;;;;AAChD,kBAAW,AAAe,wBAAI,CAAC,AAAO,SAAE,AAAM,AAAE,AAAE;AAChD,YAAI,AAAa,gBAAG,AAAC;;AACrB,cAAM,AAAM,SAAG,AAAG,AAAE;AAClB,cAAI,AAAI,OAAC,AAAS,WAAE;AAClB,AAAM;AACP;;AAED,AAAI,AAAE,iBACH,AAAI,KAAC,AAAO,AAAC,SACb,AAAK,MAAC,AAAK,AAAC,AAAE;AACb,gBAAI,EAAE,AAAa,iBAAI,AAAI,OAAC,AAAY,cAAE;AACxC,AAAM,qBAAC,AAAK,AAAC;AACd,uBACQ,AAAI,OAAC,AAAS,WAAE;AACvB,AAAM,qBAAC,IAAI,AAAK,MAAC,AAAW,AAAC,AAAC;AAC/B,aAFI,MAGA;AACH,AAAU,yBAAC,AAAM,QAAE,AAAI,OAAC,AAAY,AAAC;AACtC,AACH;AAAC,AAAC,AACN;AAAC;;AAED,AAAM,AAAE,AACV;AAAC,AAAC,AACJ,OAxBS;;AAwBR,AACF;;;;;;AASD,sCAAsC,AAAgB;AACpD,QAAM,AAAQ,WAAG,AAAI,KAAC,AAAI,KAAC,AAAQ,WAAG,AAAmB,AAAC;AAC1D,SAAO,AAAQ,WAAG,AAAkB,AAAC,AAAC,qBAAC,AAAkB,AAAC,AAAC,qBAAC,AAAQ,AACtE;AAAC","sourcesContent":["import { config as awsConfig, S3 } from \"aws-sdk\"\nimport { CreateMultipartUploadRequest } from \"aws-sdk/clients/s3\"\nimport BluebirdPromise from \"bluebird-lst\"\nimport { hashFile } from \"builder-util\"\nimport { EventEmitter } from \"events\"\nimport { createReadStream } from \"fs-extra-p\"\nimport { cpus } from \"os\"\nimport { createHash } from \"crypto\"\n\nconst MAX_PUT_OBJECT_SIZE = 5 * 1024 * 1024 * 1024\nconst MAX_MULTIPART_COUNT = 10000\nconst MIN_MULTIPART_SIZE = 5 * 1024 * 1024\nconst commonUploadSize = 15 * 1024 * 1024\n\nawsConfig.setPromisesDependency(require(\"bluebird-lst\"))\n\nexport class Uploader extends EventEmitter {\n /** @readonly */\n loaded = 0\n\n private cancelled = false\n\n readonly s3RetryCount: number\n readonly s3RetryDelay: number\n readonly multipartUploadThreshold: number\n readonly multipartUploadSize: number\n readonly multipartDownloadThreshold: number\n readonly multipartDownloadSize: number\n\n constructor(private readonly s3: S3, private readonly s3Options: CreateMultipartUploadRequest, private readonly localFile: string, readonly contentLength: number, private readonly fileContent: Buffer | null | undefined) {\n super()\n\n this.s3RetryCount = 3\n this.s3RetryDelay = 1000\n\n this.multipartUploadThreshold = (20 * 1024 * 1024)\n this.multipartUploadSize = commonUploadSize\n this.multipartDownloadThreshold = (20 * 1024 * 1024)\n this.multipartDownloadSize = commonUploadSize\n\n if (this.multipartUploadThreshold < MIN_MULTIPART_SIZE) {\n throw new Error(\"Minimum multipartUploadThreshold is 5MB.\")\n }\n if (this.multipartUploadThreshold > MAX_PUT_OBJECT_SIZE) {\n throw new Error(\"Maximum multipartUploadThreshold is 5GB.\")\n }\n if (this.multipartUploadSize < MIN_MULTIPART_SIZE) {\n throw new Error(\"Minimum multipartUploadSize is 5MB.\")\n }\n if (this.multipartUploadSize > MAX_PUT_OBJECT_SIZE) {\n throw new Error(\"Maximum multipartUploadSize is 5GB.\")\n }\n }\n\n async upload() {\n const fileContent = this.fileContent\n if (fileContent != null) {\n const hash = createHash(\"md5\")\n hash.update(fileContent)\n const md5 = hash.digest(\"base64\")\n await this.runOrRetry(() => this.putObject(md5))\n return\n }\n\n if (this.contentLength < this.multipartUploadThreshold) {\n const md5 = await hashFile(this.localFile, \"md5\")\n await this.runOrRetry(() => this.putObject(md5))\n return\n }\n\n let multipartUploadSize = this.multipartUploadSize\n if (Math.ceil(this.contentLength / multipartUploadSize) > MAX_MULTIPART_COUNT) {\n multipartUploadSize = smallestPartSizeFromFileSize(this.contentLength)\n }\n\n if (multipartUploadSize > MAX_PUT_OBJECT_SIZE) {\n throw new Error(`File size exceeds maximum object size: ${this.localFile}`)\n }\n\n const data = await this.runOrRetry(() => this.s3.createMultipartUpload(this.s3Options).promise())\n await this.multipartUpload(data.UploadId!, multipartUploadSize)\n }\n\n abort() {\n this.cancelled = true\n }\n\n private putObject(md5: string) {\n this.loaded = 0\n return new BluebirdPromise<any>((resolve, reject) => {\n this.s3.putObject({\n Body: this.fileContent || createReadStream(this.localFile),\n ContentMD5: md5,\n ...this.s3Options,\n })\n .on(\"httpUploadProgress\", progress => {\n this.loaded = progress.loaded\n this.emit(\"progress\")\n })\n .send((error, data) => {\n if (error == null) {\n resolve(data)\n }\n else {\n reject(error)\n }\n })\n })\n }\n\n private async multipartUpload(uploadId: string, multipartUploadSize: number): Promise<any> {\n let cursor = 0\n let nextPartNumber = 1\n\n const partsA: Array<any> = []\n\n const parts: Array<Part> = []\n while (cursor < this.contentLength) {\n const start = cursor\n let end = cursor + multipartUploadSize\n if (end > this.contentLength) {\n end = this.contentLength\n }\n cursor = end\n const part = {\n PartNumber: nextPartNumber++,\n }\n partsA.push(part)\n parts.push({start, end, part, md5: \"\"})\n }\n\n await BluebirdPromise.map(parts, async it => {\n // hashFile - both start and end are inclusive\n it.md5 = await hashFile(this.localFile, \"md5\", \"base64\", {start: it.start, end: it.end - 1})\n }, {concurrency: cpus().length})\n\n await BluebirdPromise.map(parts, it => this.makeUploadPart(it, uploadId), {concurrency: 4})\n return await this.runOrRetry(() => this.s3.completeMultipartUpload({\n Bucket: this.s3Options.Bucket,\n Key: this.s3Options.Key,\n UploadId: uploadId,\n MultipartUpload: {\n Parts: partsA,\n },\n }).promise()\n )\n }\n\n private makeUploadPart(part: Part, uploadId: string): Promise<any> {\n const contentLength = part.end - part.start\n return this.runOrRetry(() => {\n let partLoaded = 0\n return new BluebirdPromise((resolve, reject) => {\n this.s3.uploadPart({\n ContentLength: contentLength,\n PartNumber: part.part.PartNumber,\n UploadId: uploadId,\n Body: createReadStream(this.localFile, {start: part.start, end: part.end - 1}),\n Bucket: this.s3Options.Bucket,\n Key: this.s3Options.Key,\n ContentMD5: part.md5,\n })\n .on(\"httpUploadProgress\", progress => {\n partLoaded = progress.loaded\n this.loaded += progress.loaded\n this.emit(\"progress\")\n })\n .send((error, data) => {\n if (error == null) {\n part.part.ETag = data.ETag\n resolve(data)\n }\n else {\n this.loaded -= partLoaded\n reject(error)\n }\n })\n })\n })\n }\n\n private async runOrRetry<T>(task: () => Promise<T>): Promise<T> {\n return new BluebirdPromise<T>((resolve, reject) => {\n let attemptNumber = 0\n const tryRun = () => {\n if (this.cancelled) {\n return\n }\n\n task()\n .then(resolve)\n .catch(error => {\n if (++attemptNumber >= this.s3RetryCount) {\n reject(error)\n }\n else if (this.cancelled) {\n reject(new Error(\"cancelled\"))\n }\n else {\n setTimeout(tryRun, this.s3RetryDelay)\n }\n })\n }\n\n tryRun()\n })\n }\n}\n\ninterface Part {\n start: number\n end: number\n part: any\n md5: string\n}\n\nfunction smallestPartSizeFromFileSize(fileSize: number) {\n const partSize = Math.ceil(fileSize / MAX_MULTIPART_COUNT)\n return partSize < MIN_MULTIPART_SIZE ? MIN_MULTIPART_SIZE : partSize\n}"]}
|