| 1 | {"version":3,"sources":["../../src/utils/jobs-manager.js"],"names":["uuid","require","path","hasha","fs","pDefer","_","createContentDigest","slash","reporter","MESSAGE_TYPES","JOB_CREATED","JOB_COMPLETED","JOB_FAILED","JOB_NOT_WHITELISTED","activityForJobs","activeJobs","isListeningForMessages","hasShownIPCDisabledWarning","jobsInProcess","Map","externalJobsMap","convertPathsToAbsolute","filePath","isAbsolute","Error","createFileHash","fromFileSync","algorithm","hasActiveJobs","hasExternalJobsEnabled","process","env","ENABLE_GATSBY_EXTERNAL_JOBS","runLocalWorker","workerFn","job","ensureDir","outputDir","Promise","resolve","reject","setImmediate","inputPaths","args","err","WorkerError","listenForJobMessages","on","msg","type","payload","id","has","deferred","get","result","error","runJob","delete","runExternalWorker","set","send","promise","forceLocal","plugin","worker","posix","join","name","isLocal","warn","version","exports","createInternalJob","contentDigest","inputPathsWithContentDigest","map","internalJob","includes","inputPath","enqueueJob","phantomActivity","start","isPlainObject","message","end","getInProcessJobPromise","removeInProgressJob","waitUntilAllJobsComplete","isJobStale","areInputPathsStale","some","existsSync","fileHash","constructor","captureStackTrace"],"mappings":";;;;;AAAA,MAAMA,IAAI,GAAGC,OAAO,CAAE,SAAF,CAApB;;AACA,MAAMC,IAAI,GAAGD,OAAO,CAAE,MAAF,CAApB;;AACA,MAAME,KAAK,GAAGF,OAAO,CAAE,OAAF,CAArB;;AACA,MAAMG,EAAE,GAAGH,OAAO,CAAE,UAAF,CAAlB;;AACA,MAAMI,MAAM,GAAGJ,OAAO,CAAE,SAAF,CAAtB;;AACA,MAAMK,CAAC,GAAGL,OAAO,CAAE,QAAF,CAAjB;;AACA,MAAM;AAAEM,EAAAA,mBAAF;AAAuBC,EAAAA;AAAvB,IAAiCP,OAAO,CAAE,mBAAF,CAA9C;;AACA,MAAMQ,QAAQ,GAAGR,OAAO,CAAE,yBAAF,CAAxB;;AAEA,MAAMS,aAAa,GAAG;AACpBC,EAAAA,WAAW,EAAG,aADM;AAEpBC,EAAAA,aAAa,EAAG,eAFI;AAGpBC,EAAAA,UAAU,EAAG,YAHO;AAIpBC,EAAAA,mBAAmB,EAAG;AAJF,CAAtB;AAOA,IAAIC,eAAe,GAAG,IAAtB;AACA,IAAIC,UAAU,GAAG,CAAjB;AACA,IAAIC,sBAAsB,GAAG,KAA7B;AACA,IAAIC,0BAA0B,GAAG,KAAjC;AAEA;;AACA,MAAMC,aAAa,GAAG,IAAIC,GAAJ,EAAtB;AACA;;AACA,MAAMC,eAAe,GAAG,IAAID,GAAJ,EAAxB;AAEA;;;;;;;AAMA,MAAME,sBAAsB,GAAGC,QAAQ,IAAI;AACzC,MAAI,CAACrB,IAAI,CAACsB,UAAL,CAAgBD,QAAhB,CAAL,EAAgC;AAC9B,UAAM,IAAIE,KAAJ,CAAW,GAAEF,QAAS,8BAAtB,CAAN;AACD;;AAED,SAAOf,KAAK,CAACe,QAAD,CAAZ;AACD,CAND;AAOA;;;;;;;AAKA,MAAMG,cAAc,GAAGxB,IAAI,IAAIC,KAAK,CAACwB,YAAN,CAAmBzB,IAAnB,EAAyB;AAAE0B,EAAAA,SAAS,EAAG;AAAd,CAAzB,CAA/B;AAEA;;;;;;;;;;;;;;;;;;;;;AAqBA;;;AACA,IAAIC,aAAa,GAAG,IAApB;;AAEA,MAAMC,sBAAsB,GAAG,MAC7BC,OAAO,CAACC,GAAR,CAAYC,2BAAZ,KAA6C,MAA7C,IACAF,OAAO,CAACC,GAAR,CAAYC,2BAAZ,KAA6C,GAF/C;AAIA;;;;;;;;;;AAQA,MAAMC,cAAc,GAAG,OAAOC,QAAP,EAAiBC,GAAjB,KAAyB;AAC9C,QAAMhC,EAAE,CAACiC,SAAH,CAAaD,GAAG,CAACE,SAAjB,CAAN;AAEA,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC;AACA;AACAC,IAAAA,YAAY,CAAC,MAAM;AACjB,UAAI;AACFF,QAAAA,OAAO,CACLL,QAAQ,CAAC;AACPQ,UAAAA,UAAU,EAAEP,GAAG,CAACO,UADT;AAEPL,UAAAA,SAAS,EAAEF,GAAG,CAACE,SAFR;AAGPM,UAAAA,IAAI,EAAER,GAAG,CAACQ;AAHH,SAAD,CADH,CAAP;AAOD,OARD,CAQE,OAAOC,GAAP,EAAY;AACZJ,QAAAA,MAAM,CAAC,IAAIK,WAAJ,CAAgBD,GAAhB,CAAD,CAAN;AACD;AACF,KAZW,CAAZ;AAaD,GAhBM,CAAP;AAiBD,CApBD;;AAsBA,MAAME,oBAAoB,GAAG,MAAM;AACjChB,EAAAA,OAAO,CAACiB,EAAR,CAAY,SAAZ,EAAsBC,GAAG,IAAI;AAC3B,QACEA,GAAG,IACHA,GAAG,CAACC,IADJ,IAEAD,GAAG,CAACE,OAFJ,IAGAF,GAAG,CAACE,OAAJ,CAAYC,EAHZ,IAIA/B,eAAe,CAACgC,GAAhB,CAAoBJ,GAAG,CAACE,OAAJ,CAAYC,EAAhC,CALF,EAME;AACA,YAAM;AAAEhB,QAAAA,GAAF;AAAOkB,QAAAA;AAAP,UAAoBjC,eAAe,CAACkC,GAAhB,CAAoBN,GAAG,CAACE,OAAJ,CAAYC,EAAhC,CAA1B;;AACA,cAAQH,GAAG,CAACC,IAAZ;AACE,aAAKxC,aAAa,CAACE,aAAnB;AAAkC;AAChC0C,YAAAA,QAAQ,CAACd,OAAT,CAAiBS,GAAG,CAACE,OAAJ,CAAYK,MAA7B;AACA;AACD;;AACD,aAAK9C,aAAa,CAACG,UAAnB;AAA+B;AAC7ByC,YAAAA,QAAQ,CAACb,MAAT,CAAgB,IAAIK,WAAJ,CAAgBG,GAAG,CAACE,OAAJ,CAAYM,KAA5B,CAAhB;AACA;AACD;;AACD,aAAK/C,aAAa,CAACI,mBAAnB;AAAwC;AACtCwC,YAAAA,QAAQ,CAACd,OAAT,CAAiBkB,MAAM,CAACtB,GAAD,EAAM,IAAN,CAAvB;AACA;AACD;AAZH;;AAeAf,MAAAA,eAAe,CAACsC,MAAhB,CAAuBV,GAAG,CAACE,OAAJ,CAAYC,EAAnC;AACD;AACF,GA1BD;AA2BD,CA5BD;AA8BA;;;;;AAGA,MAAMQ,iBAAiB,GAAGxB,GAAG,IAAI;AAC/B,QAAMkB,QAAQ,GAAGjD,MAAM,EAAvB;AACAgB,EAAAA,eAAe,CAACwC,GAAhB,CAAoBzB,GAAG,CAACgB,EAAxB,EAA4B;AAC1BhB,IAAAA,GAD0B;AAE1BkB,IAAAA;AAF0B,GAA5B;AAKAvB,EAAAA,OAAO,CAAC+B,IAAR,CAAa;AACXZ,IAAAA,IAAI,EAAExC,aAAa,CAACC,WADT;AAEXwC,IAAAA,OAAO,EAAEf;AAFE,GAAb;AAKA,SAAOkB,QAAQ,CAACS,OAAhB;AACD,CAbD;AAeA;;;;;;;;;;AAQA,MAAML,MAAM,GAAG,CAACtB,GAAD,EAAM4B,UAAU,GAAG,KAAnB,KAA6B;AAC1C,QAAM;AAAEC,IAAAA;AAAF,MAAa7B,GAAnB;;AACA,MAAI;AACF,UAAM8B,MAAM,GAAGjE,OAAO,CAACC,IAAI,CAACiE,KAAL,CAAWC,IAAX,CAAgBH,MAAM,CAACzB,OAAvB,EAAiC,kBAAjC,CAAD,CAAtB;;AACA,QAAI,CAAC0B,MAAM,CAAC9B,GAAG,CAACiC,IAAL,CAAX,EAAuB;AACrB,YAAM,IAAI5C,KAAJ,CAAW,gCAA+BW,GAAG,CAACiC,IAAK,EAAnD,CAAN;AACD;;AAED,QAAI,CAACL,UAAD,IAAe,CAAC5B,GAAG,CAAC6B,MAAJ,CAAWK,OAA3B,IAAsCxC,sBAAsB,EAAhE,EAAoE;AAClE,UAAIC,OAAO,CAAC+B,IAAZ,EAAkB;AAChB,YAAI,CAAC7C,sBAAL,EAA6B;AAC3BA,UAAAA,sBAAsB,GAAG,IAAzB;AACA8B,UAAAA,oBAAoB;AACrB;;AAED,eAAOa,iBAAiB,CAACxB,GAAD,CAAxB;AACD,OAPD,MAOO;AACL;AACA,YAAI,CAAClB,0BAAL,EAAiC;AAC/BA,UAAAA,0BAA0B,GAAG,IAA7B;AACAT,UAAAA,QAAQ,CAAC8D,IAAT,CACG,+EADH;AAGD;AACF;AACF;;AACD,WAAOrC,cAAc,CAACgC,MAAM,CAAC9B,GAAG,CAACiC,IAAL,CAAP,EAAmBjC,GAAnB,CAArB;AACD,GAzBD,CAyBE,OAAOS,GAAP,EAAY;AACZ,UAAM,IAAIpB,KAAJ,CACH,uCAAsCwC,MAAM,CAACzB,OAAQ,+BAA8ByB,MAAM,CAACI,IAAK,IAAGJ,MAAM,CAACO,OAAQ,EAD9G,CAAN;AAGD;AACF,CAhCD;AAkCA;;;;;;;;;AAOAC,OAAO,CAACC,iBAAR,GAA4B,CAACtC,GAAD,EAAM6B,MAAN,KAAiB;AAC3C;AACA;AACA,MAAI7B,GAAG,CAACgB,EAAJ,IAAUhB,GAAG,CAACuC,aAAlB,EAAiC;AAC/B,WAAOvC,GAAP;AACD;;AAED,QAAM;AAAEiC,IAAAA,IAAF;AAAQ1B,IAAAA,UAAR;AAAoBL,IAAAA,SAApB;AAA+BM,IAAAA;AAA/B,MAAwCR,GAA9C,CAP2C,CAS3C;AACA;AACA;;AACA,QAAMwC,2BAA2B,GAAGjC,UAAU,CAACkC,GAAX,CAAe3E,IAAI,IAAI;AACzD,WAAO;AACLA,MAAAA,IAAI,EAAEoB,sBAAsB,CAACpB,IAAD,CADvB;AAELyE,MAAAA,aAAa,EAAEjD,cAAc,CAACxB,IAAD;AAFxB,KAAP;AAID,GALmC,CAApC;AAOA;;AACA,QAAM4E,WAAW,GAAG;AAClB1B,IAAAA,EAAE,EAAEpD,IAAI,EADU;AAElBqE,IAAAA,IAFkB;AAGlBM,IAAAA,aAAa,EAAG,EAHE;AAIlBhC,IAAAA,UAAU,EAAEiC,2BAJM;AAKlBtC,IAAAA,SAAS,EAAEhB,sBAAsB,CAACgB,SAAD,CALf;AAMlBM,IAAAA,IANkB;AAOlBqB,IAAAA,MAAM,EAAE;AACNI,MAAAA,IAAI,EAAEJ,MAAM,CAACI,IADP;AAENG,MAAAA,OAAO,EAAEP,MAAM,CAACO,OAFV;AAGNhC,MAAAA,OAAO,EAAEyB,MAAM,CAACzB,OAHV;AAIN8B,MAAAA,OAAO,EAAE,CAACL,MAAM,CAACzB,OAAP,CAAeuC,QAAf,CAAyB,gBAAzB;AAJJ;AAPU,GAApB,CApB2C,CAmC3C;;AACAD,EAAAA,WAAW,CAACH,aAAZ,GAA4BpE,mBAAmB,CAAC;AAC9C8D,IAAAA,IAAI,EAAEjC,GAAG,CAACiC,IADoC;AAE9C1B,IAAAA,UAAU,EAAEmC,WAAW,CAACnC,UAAZ,CAAuBkC,GAAvB,CACVG,SAAS,IAAIA,SAAS,CAACL,aADb,CAFkC;AAK9CrC,IAAAA,SAAS,EAAEwC,WAAW,CAACxC,SALuB;AAM9CM,IAAAA,IAAI,EAAEkC,WAAW,CAAClC,IAN4B;AAO9CqB,IAAAA,MAAM,EAAEa,WAAW,CAACb;AAP0B,GAAD,CAA/C;AAUA,SAAOa,WAAP;AACD,CA/CD;AAiDA;;;;;;;;AAMAL,OAAO,CAACQ,UAAR,GAAqB,MAAM7C,GAAN,IAAa;AAChC;AACA;AACA,MAAIjB,aAAa,CAACkC,GAAd,CAAkBjB,GAAG,CAACuC,aAAtB,CAAJ,EAA0C;AACxC,WAAOxD,aAAa,CAACoC,GAAd,CAAkBnB,GAAG,CAACuC,aAAtB,EAAqCrB,QAArC,CAA8CS,OAArD;AACD;;AAED,MAAI/C,UAAU,KAAK,CAAnB,EAAsB;AACpBa,IAAAA,aAAa,GAAGxB,MAAM,EAAtB;AACD,GAT+B,CAWhC;;;AACAW,EAAAA,UAAU;;AACV,MAAI,CAACD,eAAL,EAAsB;AACpBA,IAAAA,eAAe,GAAGN,QAAQ,CAACyE,eAAT,CAA0B,iBAA1B,CAAlB;AACAnE,IAAAA,eAAe,CAACoE,KAAhB;AACD;;AAED,QAAM7B,QAAQ,GAAGjD,MAAM,EAAvB;AACAc,EAAAA,aAAa,CAAC0C,GAAd,CAAkBzB,GAAG,CAACuC,aAAtB,EAAqC;AACnCvB,IAAAA,EAAE,EAAEhB,GAAG,CAACgB,EAD2B;AAEnCE,IAAAA;AAFmC,GAArC;;AAKA,MAAI;AACF,UAAME,MAAM,GAAG,MAAME,MAAM,CAACtB,GAAD,CAA3B,CADE,CAEF;;AACA,QAAIoB,MAAM,IAAI,IAAV,IAAkB,CAAClD,CAAC,CAAC8E,aAAF,CAAgB5B,MAAhB,CAAvB,EAAgD;AAC9C,YAAM,IAAI/B,KAAJ,CACH,oDAAmD,OAAO+B,MAAO,aAD9D,CAAN;AAGD;;AACDF,IAAAA,QAAQ,CAACd,OAAT,CAAiBgB,MAAjB;AACD,GATD,CASE,OAAOX,GAAP,EAAY;AACZ,QAAIA,GAAG,YAAYpB,KAAnB,EAA0B;AACxB6B,MAAAA,QAAQ,CAACb,MAAT,CAAgB,IAAIK,WAAJ,CAAgBD,GAAG,CAACwC,OAApB,CAAhB;AACD;;AAED/B,IAAAA,QAAQ,CAACb,MAAT,CAAgB,IAAIK,WAAJ,CAAgBD,GAAhB,CAAhB;AACD,GAfD,SAeU;AACR;AACA,QAAI,EAAE7B,UAAF,KAAiB,CAArB,EAAwB;AACtBa,MAAAA,aAAa,CAACW,OAAd;AACAzB,MAAAA,eAAe,CAACuE,GAAhB,GAFsB,CAGtB;;AACAvE,MAAAA,eAAe,GAAG,IAAlB;AACD;AACF;;AAED,SAAOuC,QAAQ,CAACS,OAAhB;AACD,CAlDD;AAoDA;;;;;;;;AAMAU,OAAO,CAACc,sBAAR,GAAiCZ,aAAa;AAAA;;AAAA,+BAC5CxD,aAAa,CAACoC,GAAd,CAAkBoB,aAAlB,CAD4C,uDAC5C,mBAAkCrB,QAAlC,CAA2CS,OADC;AAAA,CAA9C;AAGA;;;;;;;AAKAU,OAAO,CAACe,mBAAR,GAA8Bb,aAAa,IAAI;AAC7CxD,EAAAA,aAAa,CAACwC,MAAd,CAAqBgB,aAArB;AACD,CAFD;AAIA;;;;;;;AAKAF,OAAO,CAACgB,wBAAR,GAAmC,MACjC5D,aAAa,GAAGA,aAAa,CAACkC,OAAjB,GAA2BxB,OAAO,CAACC,OAAR,EAD1C;AAGA;;;;;;AAIAiC,OAAO,CAACiB,UAAR,GAAqBtD,GAAG,IAAI;AAC1B,QAAMuD,kBAAkB,GAAGvD,GAAG,CAACO,UAAJ,CAAeiD,IAAf,CAAoBZ,SAAS,IAAI;AAC1D;AACA,QAAI,CAAC5E,EAAE,CAACyF,UAAH,CAAcb,SAAS,CAAC9E,IAAxB,CAAL,EAAoC;AAClC,aAAO,IAAP;AACD,KAJyD,CAM1D;;;AACA,UAAM4F,QAAQ,GAAGpE,cAAc,CAACsD,SAAS,CAAC9E,IAAX,CAA/B;AACA,WAAO4F,QAAQ,KAAKd,SAAS,CAACL,aAA9B;AACD,GAT0B,CAA3B;AAWA,SAAOgB,kBAAP;AACD,CAbD;;AAeO,MAAM7C,WAAN,SAA0BrB,KAA1B,CAAgC;AACrCsE,EAAAA,WAAW,CAACV,OAAD,EAAU;AACnB,UAAMA,OAAN;AACA,SAAKhB,IAAL,GAAa,aAAb;AAEA5C,IAAAA,KAAK,CAACuE,iBAAN,CAAwB,IAAxB,EAA8BlD,WAA9B;AACD;;AANoC","sourcesContent":["const uuid = require(`uuid/v4`)\nconst path = require(`path`)\nconst hasha = require(`hasha`)\nconst fs = require(`fs-extra`)\nconst pDefer = require(`p-defer`)\nconst _ = require(`lodash`)\nconst { createContentDigest, slash } = require(`gatsby-core-utils`)\nconst reporter = require(`gatsby-cli/lib/reporter`)\n\nconst MESSAGE_TYPES = {\n JOB_CREATED: `JOB_CREATED`,\n JOB_COMPLETED: `JOB_COMPLETED`,\n JOB_FAILED: `JOB_FAILED`,\n JOB_NOT_WHITELISTED: `JOB_NOT_WHITELISTED`,\n}\n\nlet activityForJobs = null\nlet activeJobs = 0\nlet isListeningForMessages = false\nlet hasShownIPCDisabledWarning = false\n\n/** @type {Map<string, {id: string, deferred: pDefer.DeferredPromise<any>}>} */\nconst jobsInProcess = new Map()\n/** @type {Map<string, {job: InternalJob, deferred: pDefer.DeferredPromise<any>}>} */\nconst externalJobsMap = new Map()\n\n/**\n * We want to use absolute paths to make sure they are on the filesystem\n *\n * @param {string} filePath\n * @return {string}\n */\nconst convertPathsToAbsolute = filePath => {\n if (!path.isAbsolute(filePath)) {\n throw new Error(`${filePath} should be an absolute path.`)\n }\n\n return slash(filePath)\n}\n/**\n * Get contenthash of a file\n *\n * @param {string} path\n */\nconst createFileHash = path => hasha.fromFileSync(path, { algorithm: `sha1` })\n\n/**\n * @typedef BaseJobInterface\n * @property {string} name\n * @property {string} outputDir,\n * @property {Record<string, any>} args\n\n * @typedef JobInputInterface\n * @property {string[]} inputPaths\n * @property {{name: string, version: string, resolve: string}} plugin\n\n * @typedef InternalJobInterface\n * @property {string} id\n * @property {string} contentDigest\n * @property {{path: string, contentDigest: string}[]} inputPaths\n * @property {{name: string, version: string, resolve: string, isLocal: boolean}} plugin\n *\n * I know this sucks but this is the only way to do it properly in jsdoc..\n * @typedef {BaseJobInterface & JobInputInterface} JobInput\n * @typedef {BaseJobInterface & InternalJobInterface} InternalJob\n */\n\n/** @type {pDefer.DeferredPromise<void>|null} */\nlet hasActiveJobs = null\n\nconst hasExternalJobsEnabled = () =>\n process.env.ENABLE_GATSBY_EXTERNAL_JOBS === `true` ||\n process.env.ENABLE_GATSBY_EXTERNAL_JOBS === `1`\n\n/**\n * Get the local worker function and execute it on the user's machine\n *\n * @template T\n * @param {function({ inputPaths: InternalJob[\"inputPaths\"], outputDir: InternalJob[\"outputDir\"], args: InternalJob[\"args\"]}): T} workerFn\n * @param {InternalJob} job\n * @return {Promise<T>}\n */\nconst runLocalWorker = async (workerFn, job) => {\n await fs.ensureDir(job.outputDir)\n\n return new Promise((resolve, reject) => {\n // execute worker nextTick\n // TODO should we think about threading/queueing here?\n setImmediate(() => {\n try {\n resolve(\n workerFn({\n inputPaths: job.inputPaths,\n outputDir: job.outputDir,\n args: job.args,\n })\n )\n } catch (err) {\n reject(new WorkerError(err))\n }\n })\n })\n}\n\nconst listenForJobMessages = () => {\n process.on(`message`, msg => {\n if (\n msg &&\n msg.type &&\n msg.payload &&\n msg.payload.id &&\n externalJobsMap.has(msg.payload.id)\n ) {\n const { job, deferred } = externalJobsMap.get(msg.payload.id)\n switch (msg.type) {\n case MESSAGE_TYPES.JOB_COMPLETED: {\n deferred.resolve(msg.payload.result)\n break\n }\n case MESSAGE_TYPES.JOB_FAILED: {\n deferred.reject(new WorkerError(msg.payload.error))\n break\n }\n case MESSAGE_TYPES.JOB_NOT_WHITELISTED: {\n deferred.resolve(runJob(job, true))\n break\n }\n }\n\n externalJobsMap.delete(msg.payload.id)\n }\n })\n}\n\n/**\n * @param {InternalJob} job\n */\nconst runExternalWorker = job => {\n const deferred = pDefer()\n externalJobsMap.set(job.id, {\n job,\n deferred,\n })\n\n process.send({\n type: MESSAGE_TYPES.JOB_CREATED,\n payload: job,\n })\n\n return deferred.promise\n}\n\n/**\n * Make sure we have everything we need to run a job\n * If we do, run it locally.\n * TODO add external job execution through ipc\n *\n * @param {InternalJob} job\n * @return {Promise<object>}\n */\nconst runJob = (job, forceLocal = false) => {\n const { plugin } = job\n try {\n const worker = require(path.posix.join(plugin.resolve, `gatsby-worker.js`))\n if (!worker[job.name]) {\n throw new Error(`No worker function found for ${job.name}`)\n }\n\n if (!forceLocal && !job.plugin.isLocal && hasExternalJobsEnabled()) {\n if (process.send) {\n if (!isListeningForMessages) {\n isListeningForMessages = true\n listenForJobMessages()\n }\n\n return runExternalWorker(job)\n } else {\n // only show the offloading warning once\n if (!hasShownIPCDisabledWarning) {\n hasShownIPCDisabledWarning = true\n reporter.warn(\n `Offloading of a job failed as IPC could not be detected. Running job locally.`\n )\n }\n }\n }\n return runLocalWorker(worker[job.name], job)\n } catch (err) {\n throw new Error(\n `We couldn't find a gatsby-worker.js(${plugin.resolve}/gatsby-worker.js) file for ${plugin.name}@${plugin.version}`\n )\n }\n}\n\n/**\n * Create an internal job object\n *\n * @param {JobInput|InternalJob} job\n * @param {{name: string, version: string, resolve: string}} plugin\n * @return {InternalJob}\n */\nexports.createInternalJob = (job, plugin) => {\n // It looks like we already have an augmented job so we shouldn't redo this work\n // @ts-ignore\n if (job.id && job.contentDigest) {\n return job\n }\n\n const { name, inputPaths, outputDir, args } = job\n\n // TODO see if we can make this async, filehashing might be expensive to wait for\n // currently this needs to be sync as we could miss jobs to have been scheduled and\n // are still processing their hashes\n const inputPathsWithContentDigest = inputPaths.map(path => {\n return {\n path: convertPathsToAbsolute(path),\n contentDigest: createFileHash(path),\n }\n })\n\n /** @type {InternalJob} */\n const internalJob = {\n id: uuid(),\n name,\n contentDigest: ``,\n inputPaths: inputPathsWithContentDigest,\n outputDir: convertPathsToAbsolute(outputDir),\n args,\n plugin: {\n name: plugin.name,\n version: plugin.version,\n resolve: plugin.resolve,\n isLocal: !plugin.resolve.includes(`/node_modules/`),\n },\n }\n\n // generate a contentDigest based on all parameters including file content\n internalJob.contentDigest = createContentDigest({\n name: job.name,\n inputPaths: internalJob.inputPaths.map(\n inputPath => inputPath.contentDigest\n ),\n outputDir: internalJob.outputDir,\n args: internalJob.args,\n plugin: internalJob.plugin,\n })\n\n return internalJob\n}\n\n/**\n * Creates a job\n *\n * @param {InternalJob} job\n * @return {Promise<object>}\n */\nexports.enqueueJob = async job => {\n // When we already have a job that's executing, return the same promise.\n // we have another check in our createJobV2 action to return jobs that have been done in a previous gatsby run\n if (jobsInProcess.has(job.contentDigest)) {\n return jobsInProcess.get(job.contentDigest).deferred.promise\n }\n\n if (activeJobs === 0) {\n hasActiveJobs = pDefer()\n }\n\n // Bump active jobs\n activeJobs++\n if (!activityForJobs) {\n activityForJobs = reporter.phantomActivity(`Running jobs v2`)\n activityForJobs.start()\n }\n\n const deferred = pDefer()\n jobsInProcess.set(job.contentDigest, {\n id: job.id,\n deferred,\n })\n\n try {\n const result = await runJob(job)\n // this check is to keep our worker results consistent for cloud\n if (result != null && !_.isPlainObject(result)) {\n throw new Error(\n `Result of a worker should be an object, type of \"${typeof result}\" was given`\n )\n }\n deferred.resolve(result)\n } catch (err) {\n if (err instanceof Error) {\n deferred.reject(new WorkerError(err.message))\n }\n\n deferred.reject(new WorkerError(err))\n } finally {\n // when all jobs are done we end the activity\n if (--activeJobs === 0) {\n hasActiveJobs.resolve()\n activityForJobs.end()\n // eslint-disable-next-line require-atomic-updates\n activityForJobs = null\n }\n }\n\n return deferred.promise\n}\n\n/**\n * Get in progress job promise\n *\n * @param {string} contentDigest\n * @return {Promise<void>}\n */\nexports.getInProcessJobPromise = contentDigest =>\n jobsInProcess.get(contentDigest)?.deferred.promise\n\n/**\n * Remove a job from our inProgressQueue to reduce memory usage\n *\n * @param {string} contentDigest\n */\nexports.removeInProgressJob = contentDigest => {\n jobsInProcess.delete(contentDigest)\n}\n\n/**\n * Wait for all processing jobs to have finished\n *\n * @return {Promise<void>}\n */\nexports.waitUntilAllJobsComplete = () =>\n hasActiveJobs ? hasActiveJobs.promise : Promise.resolve()\n\n/**\n * @param {Partial<InternalJob> & {inputPaths: InternalJob['inputPaths']}} job\n * @return {boolean}\n */\nexports.isJobStale = job => {\n const areInputPathsStale = job.inputPaths.some(inputPath => {\n // does the inputPath still exists?\n if (!fs.existsSync(inputPath.path)) {\n return true\n }\n\n // check if we're talking about the same file\n const fileHash = createFileHash(inputPath.path)\n return fileHash !== inputPath.contentDigest\n })\n\n return areInputPathsStale\n}\n\nexport class WorkerError extends Error {\n constructor(message) {\n super(message)\n this.name = `WorkerError`\n\n Error.captureStackTrace(this, WorkerError)\n }\n}\n"],"file":"jobs-manager.js"} |