{"version":3,"sources":["../src/index.ts","../src/storage.ts"],"sourcesContent":["export * as googleStorage from '@google-cloud/storage'\nexport {\n HUBMETRICS_FILE_FOLDER,\n getStorageClient,\n getFileObject,\n getFileACL,\n setFileMetadata,\n copyFile,\n downloadFile,\n getFileHash,\n uploadFile,\n deleteFiles,\n concatenateSubFiles\n} from './storage';\n","import { Storage, File, Bucket, CopyOptions, GetAclResponse } from '@google-cloud/storage'\nimport { v4 as uuid } from 'uuid'\nimport promiseRetry from 'promise-retry'\n\nimport { shouldRetryGoogleRequest } from '@arcane-utils/error'\n\nconst RETRY_CONFIG = {\n retries: 5,\n factor: 2,\n minTimeout: 1000\n}\nexport const HUBMETRICS_FILE_FOLDER: string = 'hubmetrics_75012'\n\nexport const getStorageClient = (gcpProject: string, serviceAccount: string): Storage => {\n return new Storage({\n projectId: gcpProject,\n keyFilename: serviceAccount\n })\n}\n\nexport const getFileObject = (bucketName: string, fileName: string, storageClient: Storage): File => {\n return storageClient.bucket(bucketName).file(fileName)\n}\n\nexport const getFileACL = async (file: File, aclQuery: any): Promise => {\n return promiseRetry(async (retry, number) => {\n try {\n return await file.acl.get(aclQuery)\n } catch (error: any) {\n if (shouldRetryGoogleRequest(error.code)) {\n console.log(`Could not get ACL: ${error.message}. Attempt is ${number}. Trying again...`)\n retry(error)\n }\n throw error\n }\n }, RETRY_CONFIG)\n}\n\n/**\n * Set file metadata\n * @example\n * metadata = {\n * metadata: {\n * key1: \"value1\",\n * key2: \"value2\",\n * }\n * }\n */\nexport const setFileMetadata = async (file: File, metadata: any): Promise => {\n return promiseRetry(async (retry, number) => {\n try {\n await file.setMetadata(metadata)\n } catch (error: any) {\n if (shouldRetryGoogleRequest(error.code)) {\n console.log(`Could not set metadata: ${error.message}. Attempt is ${number}. Trying again...`)\n retry(error)\n }\n throw error\n }\n }, RETRY_CONFIG)\n}\n\nexport const copyFile = async (\n sourceFile: File,\n destinationFile: string | File | Bucket,\n options?: CopyOptions\n): Promise => {\n return promiseRetry(async (retry, number) => {\n try {\n await sourceFile.copy(destinationFile, options)\n } catch (error: any) {\n if (shouldRetryGoogleRequest(error.code)) {\n console.log(`Could not copy ${sourceFile.name} to ${destinationFile}: ${error.message}. Attempt is ${number}. Trying again...`)\n retry(error)\n }\n throw error\n }\n }, RETRY_CONFIG)\n}\n\nexport const downloadFile = async (file: File, destination: string): Promise => {\n return promiseRetry(async (retry, number) => {\n try {\n await file.download({\n destination\n })\n } catch (error: any) {\n if (shouldRetryGoogleRequest(error.code)) {\n console.log(`Could not download ${file} to ${destination}: ${error.message}. Attempt is ${number}. Trying again...`)\n retry(error)\n }\n }\n }, RETRY_CONFIG)\n}\n\nexport const getFileHash = async (file: File): Promise => {\n return promiseRetry(async (retry, number) => {\n try {\n const [metadata] = await file.getMetadata()\n console.log(`Crc32c: ${metadata.crc32c}`)\n const { crc32c } = metadata\n return crc32c\n } catch (error: any) {\n if (shouldRetryGoogleRequest(error.code)) {\n console.log(`Could not getFileMetadata ${file} : ${error.message}. Attempt is ${number}. Trying again...`)\n retry(error)\n }\n }\n }, RETRY_CONFIG)\n}\n\nexport const uploadFile = async (bucket: Bucket, filePath: string, destFileName: string | File) => {\n return promiseRetry(async (retry) => {\n try {\n await bucket.upload(filePath, {\n destination: destFileName\n })\n } catch (error) {\n console.log(`${filePath} uploaded to ${bucket.name}`)\n retry(error)\n }\n }, RETRY_CONFIG)\n}\n\nexport const deleteFiles = async (filesList: File[]): Promise => {\n return promiseRetry(async (retry) => {\n try {\n await Promise.all(filesList.map((file) => file.delete()))\n } catch (error) {\n retry(error)\n }\n }, RETRY_CONFIG)\n}\n\n/**\n *\n * @param filesList list of files to combine\n * @param filePath the result file path\n * @param bucket a working bucket used for file concatenation (not datalake bucket)\n * @param metadata optional metadata to set on the result file\n * @returns\n */\nexport const concatenateSubFiles = async (filesList: File[], outputFile: File, bucket: Bucket, logger: any = console, metadata?: Object): Promise => {\n return promiseRetry(async (retry, number) => {\n let temporaryFolder = 1\n let tempFilesList: File[] = [...filesList]\n let fileToDelete: File[] = []\n\n const executionId = uuid()\n\n while (tempFilesList.length > 32) {\n let index = 1\n let filesToCombine: File[] = []\n for (let idx = 0; idx < tempFilesList.length; idx += 1) {\n filesToCombine.push(tempFilesList[idx])\n if ((filesToCombine.length === 32) || (idx === filesList.length - 1)) {\n await bucket.combine(filesToCombine, `${outputFile.name}_${executionId}/temp_combine_${temporaryFolder}/file_combined_${index}`)\n index += 1\n filesToCombine = []\n }\n }\n [tempFilesList] = await bucket.getFiles({ prefix: `${outputFile.name}_${executionId}/temp_combine_${temporaryFolder}` })\n // We only want to delete temp files but not files passed as arg in filesList\n fileToDelete = fileToDelete.concat([...tempFilesList])\n temporaryFolder += 1\n }\n try {\n await bucket.combine(tempFilesList, outputFile)\n if (metadata) {\n outputFile.setMetadata({\n metadata \n })\n }\n await deleteFiles(fileToDelete)\n return outputFile\n\n } catch (error: any) {\n if (shouldRetryGoogleRequest(error.code)) {\n logger.info(`Could not concatenate sub files. Attempt is ${number}. Trying again...`)\n logger.info(error)\n return retry(error)\n }\n throw error\n }\n }, RETRY_CONFIG)\n}\n"],"mappings":"6iBAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,4BAAAE,EAAA,wBAAAC,EAAA,aAAAC,EAAA,gBAAAC,EAAA,iBAAAC,EAAA,eAAAC,EAAA,gBAAAC,EAAA,kBAAAC,EAAA,qBAAAC,EAAA,kBAAAC,EAAA,oBAAAC,EAAA,eAAAC,IAAA,eAAAC,EAAAd,GAAA,IAAAW,EAA+B,sCCA/B,IAAAI,EAAmE,iCACnEC,EAA2B,gBAC3BC,EAAyB,8BAEzBC,EAAyC,+BAEnCC,EAAe,CACnB,QAAS,EACT,OAAQ,EACR,WAAY,GACd,EACaC,EAAiC,mBAEjCC,EAAmB,CAACC,EAAoBC,IAC5C,IAAI,UAAQ,CACjB,UAAWD,EACX,YAAaC,CACf,CAAC,EAGUC,EAAgB,CAACC,EAAoBC,EAAkBC,IAC3DA,EAAc,OAAOF,CAAU,EAAE,KAAKC,CAAQ,EAG1CE,EAAa,MAAOC,EAAYC,OACpC,EAAAC,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,OAAO,MAAMJ,EAAK,IAAI,IAAIC,CAAQ,CACpC,OAASI,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,sBAAsBA,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACxFD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGf,CAAY,EAaJgB,EAAkB,MAAON,EAAYO,OACzC,EAAAL,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMJ,EAAK,YAAYO,CAAQ,CACjC,OAASF,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,2BAA2BA,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EAC7FD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGf,CAAY,EAGJkB,EAAW,MACtBC,EACAC,EACAC,OAEO,EAAAT,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMK,EAAW,KAAKC,EAAiBC,CAAO,CAChD,OAASN,EAAY,CACnB,QAAI,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,kBAAkBI,EAAW,IAAI,OAAOC,CAAe,KAAKL,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EAC9HD,EAAME,CAAK,GAEPA,CACR,CACF,EAAGf,CAAY,EAGJsB,EAAe,MAAOZ,EAAYa,OACtC,EAAAX,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,MAAMJ,EAAK,SAAS,CAClB,YAAAa,CACF,CAAC,CACH,OAASR,EAAY,IACf,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,sBAAsBL,CAAI,OAAOa,CAAW,KAAKR,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACnHD,EAAME,CAAK,EAEf,CACF,EAAGf,CAAY,EAGJwB,EAAc,MAAOd,MACzB,EAAAE,SAAa,MAAOC,EAAOC,IAAW,CAC3C,GAAI,CACF,GAAM,CAACG,CAAQ,EAAI,MAAMP,EAAK,YAAY,EAC1C,QAAQ,IAAI,WAAWO,EAAS,MAAM,EAAE,EACxC,GAAM,CAAE,OAAAQ,CAAO,EAAIR,EACnB,OAAOQ,CACT,OAASV,EAAY,IACf,4BAAyBA,EAAM,IAAI,IACrC,QAAQ,IAAI,6BAA6BL,CAAI,MAAMK,EAAM,OAAO,gBAAgBD,CAAM,mBAAmB,EACzGD,EAAME,CAAK,EAEf,CACF,EAAGf,CAAY,EAGJ0B,EAAa,MAAOC,EAAgBC,EAAkBC,OAC1D,EAAAjB,SAAa,MAAOC,GAAU,CACnC,GAAI,CACF,MAAMc,EAAO,OAAOC,EAAU,CAC5B,YAAaC,CACf,CAAC,CACH,OAASd,EAAO,CACd,QAAQ,IAAI,GAAGa,CAAQ,gBAAgBD,EAAO,IAAI,EAAE,EACpDd,EAAME,CAAK,CACb,CACF,EAAGf,CAAY,EAGJ8B,EAAc,MAAOC,MACzB,EAAAnB,SAAa,MAAOC,GAAU,CACnC,GAAI,CACF,MAAM,QAAQ,IAAIkB,EAAU,IAAKrB,GAASA,EAAK,OAAO,CAAC,CAAC,CAC1D,OAASK,EAAO,CACdF,EAAME,CAAK,CACb,CACF,EAAGf,CAAY,EAWJgC,EAAsB,MAAOD,EAAmBE,EAAkBN,EAAgBO,EAAc,QAASjB,OAC7G,EAAAL,SAAa,MAAOC,EAAOC,IAAW,CAC3C,IAAIqB,EAAkB,EAClBC,EAAwB,CAAC,GAAGL,CAAS,EACrCM,EAAuB,CAAC,EAEtBC,KAAc,EAAAC,IAAK,EAEzB,KAAOH,EAAc,OAAS,IAAI,CAChC,IAAII,EAAQ,EACRC,EAAyB,CAAC,EAC9B,QAASC,EAAM,EAAGA,EAAMN,EAAc,OAAQM,GAAO,EACnDD,EAAe,KAAKL,EAAcM,CAAG,CAAC,GACjCD,EAAe,SAAW,IAAQC,IAAQX,EAAU,OAAS,KAChE,MAAMJ,EAAO,QAAQc,EAAgB,GAAGR,EAAW,IAAI,IAAIK,CAAW,iBAAiBH,CAAe,kBAAkBK,CAAK,EAAE,EAC/HA,GAAS,EACTC,EAAiB,CAAC,GAGtB,CAACL,CAAa,EAAI,MAAMT,EAAO,SAAS,CAAE,OAAQ,GAAGM,EAAW,IAAI,IAAIK,CAAW,iBAAiBH,CAAe,EAAG,CAAC,EAEvHE,EAAeA,EAAa,OAAO,CAAC,GAAGD,CAAa,CAAC,EACrDD,GAAmB,CACrB,CACA,GAAI,CACF,aAAMR,EAAO,QAAQS,EAAeH,CAAU,EAC1ChB,GACFgB,EAAW,YAAY,CACrB,SAAAhB,CACF,CAAC,EAEH,MAAMa,EAAYO,CAAY,EACvBJ,CAET,OAASlB,EAAY,CACnB,MAAI,4BAAyBA,EAAM,IAAI,EACrC,OAAAmB,EAAO,KAAK,+CAA+CpB,CAAM,mBAAmB,EACpFoB,EAAO,KAAKnB,CAAK,EACVF,EAAME,CAAK,EAEpB,MAAMA,CACR,CACF,EAAGf,CAAY","names":["src_exports","__export","HUBMETRICS_FILE_FOLDER","concatenateSubFiles","copyFile","deleteFiles","downloadFile","getFileACL","getFileHash","getFileObject","getStorageClient","googleStorage","setFileMetadata","uploadFile","__toCommonJS","import_storage","import_uuid","import_promise_retry","import_error","RETRY_CONFIG","HUBMETRICS_FILE_FOLDER","getStorageClient","gcpProject","serviceAccount","getFileObject","bucketName","fileName","storageClient","getFileACL","file","aclQuery","promiseRetry","retry","number","error","setFileMetadata","metadata","copyFile","sourceFile","destinationFile","options","downloadFile","destination","getFileHash","crc32c","uploadFile","bucket","filePath","destFileName","deleteFiles","filesList","concatenateSubFiles","outputFile","logger","temporaryFolder","tempFilesList","fileToDelete","executionId","uuid","index","filesToCombine","idx"]}