1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 | "use strict";
|
7 |
|
8 | const asyncLib = require("neo-async");
|
9 | const { SyncHook, MultiHook } = require("tapable");
|
10 |
|
11 | const ConcurrentCompilationError = require("./ConcurrentCompilationError");
|
12 | const MultiStats = require("./MultiStats");
|
13 | const MultiWatching = require("./MultiWatching");
|
14 | const ArrayQueue = require("./util/ArrayQueue");
|
15 |
|
16 |
|
17 |
|
18 |
|
19 |
|
20 |
|
21 |
|
22 |
|
23 |
|
24 |
|
25 |
|
26 |
|
27 |
|
28 |
|
29 |
|
30 |
|
31 |
|
32 |
|
33 |
|
34 |
|
35 |
|
36 |
|
37 |
|
38 |
|
39 |
|
40 |
|
41 |
|
42 |
|
43 |
|
44 |
|
45 | module.exports = class MultiCompiler {
|
46 | |
47 |
|
48 |
|
49 |
|
50 | constructor(compilers, options) {
|
51 | if (!Array.isArray(compilers)) {
|
52 | compilers = Object.keys(compilers).map(name => {
|
53 | compilers[name].name = name;
|
54 | return compilers[name];
|
55 | });
|
56 | }
|
57 |
|
58 | this.hooks = Object.freeze({
|
59 |
|
60 | done: new SyncHook(["stats"]),
|
61 |
|
62 | invalid: new MultiHook(compilers.map(c => c.hooks.invalid)),
|
63 |
|
64 | run: new MultiHook(compilers.map(c => c.hooks.run)),
|
65 |
|
66 | watchClose: new SyncHook([]),
|
67 |
|
68 | watchRun: new MultiHook(compilers.map(c => c.hooks.watchRun)),
|
69 |
|
70 | infrastructureLog: new MultiHook(
|
71 | compilers.map(c => c.hooks.infrastructureLog)
|
72 | )
|
73 | });
|
74 | this.compilers = compilers;
|
75 |
|
76 | this._options = {
|
77 | parallelism: options.parallelism || Infinity
|
78 | };
|
79 |
|
80 | this.dependencies = new WeakMap();
|
81 | this.running = false;
|
82 |
|
83 |
|
84 | const compilerStats = this.compilers.map(() => null);
|
85 | let doneCompilers = 0;
|
86 | for (let index = 0; index < this.compilers.length; index++) {
|
87 | const compiler = this.compilers[index];
|
88 | const compilerIndex = index;
|
89 | let compilerDone = false;
|
90 |
|
91 | compiler.hooks.done.tap("MultiCompiler", stats => {
|
92 | if (!compilerDone) {
|
93 | compilerDone = true;
|
94 | doneCompilers++;
|
95 | }
|
96 | compilerStats[compilerIndex] = stats;
|
97 | if (doneCompilers === this.compilers.length) {
|
98 | this.hooks.done.call(new MultiStats(compilerStats));
|
99 | }
|
100 | });
|
101 |
|
102 | compiler.hooks.invalid.tap("MultiCompiler", () => {
|
103 | if (compilerDone) {
|
104 | compilerDone = false;
|
105 | doneCompilers--;
|
106 | }
|
107 | });
|
108 | }
|
109 | }
|
110 |
|
111 | get options() {
|
112 | return Object.assign(
|
113 | this.compilers.map(c => c.options),
|
114 | this._options
|
115 | );
|
116 | }
|
117 |
|
118 | get outputPath() {
|
119 | let commonPath = this.compilers[0].outputPath;
|
120 | for (const compiler of this.compilers) {
|
121 | while (
|
122 | compiler.outputPath.indexOf(commonPath) !== 0 &&
|
123 | /[/\\]/.test(commonPath)
|
124 | ) {
|
125 | commonPath = commonPath.replace(/[/\\][^/\\]*$/, "");
|
126 | }
|
127 | }
|
128 |
|
129 | if (!commonPath && this.compilers[0].outputPath[0] === "/") return "/";
|
130 | return commonPath;
|
131 | }
|
132 |
|
133 | get inputFileSystem() {
|
134 | throw new Error("Cannot read inputFileSystem of a MultiCompiler");
|
135 | }
|
136 |
|
137 | get outputFileSystem() {
|
138 | throw new Error("Cannot read outputFileSystem of a MultiCompiler");
|
139 | }
|
140 |
|
141 | get watchFileSystem() {
|
142 | throw new Error("Cannot read watchFileSystem of a MultiCompiler");
|
143 | }
|
144 |
|
145 | get intermediateFileSystem() {
|
146 | throw new Error("Cannot read outputFileSystem of a MultiCompiler");
|
147 | }
|
148 |
|
149 | |
150 |
|
151 |
|
152 | set inputFileSystem(value) {
|
153 | for (const compiler of this.compilers) {
|
154 | compiler.inputFileSystem = value;
|
155 | }
|
156 | }
|
157 |
|
158 | |
159 |
|
160 |
|
161 | set outputFileSystem(value) {
|
162 | for (const compiler of this.compilers) {
|
163 | compiler.outputFileSystem = value;
|
164 | }
|
165 | }
|
166 |
|
167 | |
168 |
|
169 |
|
170 | set watchFileSystem(value) {
|
171 | for (const compiler of this.compilers) {
|
172 | compiler.watchFileSystem = value;
|
173 | }
|
174 | }
|
175 |
|
176 | |
177 |
|
178 |
|
179 | set intermediateFileSystem(value) {
|
180 | for (const compiler of this.compilers) {
|
181 | compiler.intermediateFileSystem = value;
|
182 | }
|
183 | }
|
184 |
|
185 | getInfrastructureLogger(name) {
|
186 | return this.compilers[0].getInfrastructureLogger(name);
|
187 | }
|
188 |
|
189 | |
190 |
|
191 |
|
192 |
|
193 |
|
194 | setDependencies(compiler, dependencies) {
|
195 | this.dependencies.set(compiler, dependencies);
|
196 | }
|
197 |
|
198 | |
199 |
|
200 |
|
201 |
|
202 | validateDependencies(callback) {
|
203 |
|
204 | const edges = new Set();
|
205 |
|
206 | const missing = [];
|
207 | const targetFound = compiler => {
|
208 | for (const edge of edges) {
|
209 | if (edge.target === compiler) {
|
210 | return true;
|
211 | }
|
212 | }
|
213 | return false;
|
214 | };
|
215 | const sortEdges = (e1, e2) => {
|
216 | return (
|
217 | e1.source.name.localeCompare(e2.source.name) ||
|
218 | e1.target.name.localeCompare(e2.target.name)
|
219 | );
|
220 | };
|
221 | for (const source of this.compilers) {
|
222 | const dependencies = this.dependencies.get(source);
|
223 | if (dependencies) {
|
224 | for (const dep of dependencies) {
|
225 | const target = this.compilers.find(c => c.name === dep);
|
226 | if (!target) {
|
227 | missing.push(dep);
|
228 | } else {
|
229 | edges.add({
|
230 | source,
|
231 | target
|
232 | });
|
233 | }
|
234 | }
|
235 | }
|
236 | }
|
237 |
|
238 | const errors = missing.map(m => `Compiler dependency \`${m}\` not found.`);
|
239 | const stack = this.compilers.filter(c => !targetFound(c));
|
240 | while (stack.length > 0) {
|
241 | const current = stack.pop();
|
242 | for (const edge of edges) {
|
243 | if (edge.source === current) {
|
244 | edges.delete(edge);
|
245 | const target = edge.target;
|
246 | if (!targetFound(target)) {
|
247 | stack.push(target);
|
248 | }
|
249 | }
|
250 | }
|
251 | }
|
252 | if (edges.size > 0) {
|
253 |
|
254 | const lines = Array.from(edges)
|
255 | .sort(sortEdges)
|
256 | .map(edge => `${edge.source.name} -> ${edge.target.name}`);
|
257 | lines.unshift("Circular dependency found in compiler dependencies.");
|
258 | errors.unshift(lines.join("\n"));
|
259 | }
|
260 | if (errors.length > 0) {
|
261 | const message = errors.join("\n");
|
262 | callback(new Error(message));
|
263 | return false;
|
264 | }
|
265 | return true;
|
266 | }
|
267 |
|
268 |
|
269 | |
270 |
|
271 |
|
272 |
|
273 |
|
274 |
|
275 |
|
276 | runWithDependencies(compilers, fn, callback) {
|
277 | const fulfilledNames = new Set();
|
278 | let remainingCompilers = compilers;
|
279 | const isDependencyFulfilled = d => fulfilledNames.has(d);
|
280 | const getReadyCompilers = () => {
|
281 | let readyCompilers = [];
|
282 | let list = remainingCompilers;
|
283 | remainingCompilers = [];
|
284 | for (const c of list) {
|
285 | const dependencies = this.dependencies.get(c);
|
286 | const ready =
|
287 | !dependencies || dependencies.every(isDependencyFulfilled);
|
288 | if (ready) {
|
289 | readyCompilers.push(c);
|
290 | } else {
|
291 | remainingCompilers.push(c);
|
292 | }
|
293 | }
|
294 | return readyCompilers;
|
295 | };
|
296 | const runCompilers = callback => {
|
297 | if (remainingCompilers.length === 0) return callback();
|
298 | asyncLib.map(
|
299 | getReadyCompilers(),
|
300 | (compiler, callback) => {
|
301 | fn(compiler, err => {
|
302 | if (err) return callback(err);
|
303 | fulfilledNames.add(compiler.name);
|
304 | runCompilers(callback);
|
305 | });
|
306 | },
|
307 | callback
|
308 | );
|
309 | };
|
310 | runCompilers(callback);
|
311 | }
|
312 |
|
313 | |
314 |
|
315 |
|
316 |
|
317 |
|
318 |
|
319 |
|
320 | _runGraph(setup, run, callback) {
|
321 |
|
322 |
|
323 |
|
324 |
|
325 |
|
326 |
|
327 |
|
328 |
|
329 |
|
330 |
|
331 |
|
332 |
|
333 |
|
334 |
|
335 |
|
336 | const nodes = this.compilers.map(compiler => ({
|
337 | compiler,
|
338 | result: undefined,
|
339 | state: "blocked",
|
340 | children: [],
|
341 | parents: []
|
342 | }));
|
343 |
|
344 | const compilerToNode = new Map();
|
345 | for (const node of nodes) compilerToNode.set(node.compiler.name, node);
|
346 | for (const node of nodes) {
|
347 | const dependencies = this.dependencies.get(node.compiler);
|
348 | if (!dependencies) continue;
|
349 | for (const dep of dependencies) {
|
350 | const parent = compilerToNode.get(dep);
|
351 | node.parents.push(parent);
|
352 | parent.children.push(node);
|
353 | }
|
354 | }
|
355 |
|
356 | const queue = new ArrayQueue();
|
357 | for (const node of nodes) {
|
358 | if (node.parents.length === 0) {
|
359 | node.state = "queued";
|
360 | queue.enqueue(node);
|
361 | }
|
362 | }
|
363 | let errored = false;
|
364 | let running = 0;
|
365 | const parallelism = this._options.parallelism;
|
366 | |
367 |
|
368 |
|
369 |
|
370 |
|
371 |
|
372 | const nodeDone = (node, err, stats) => {
|
373 | if (errored) return;
|
374 | if (err) {
|
375 | errored = true;
|
376 | return asyncLib.each(
|
377 | nodes,
|
378 | (node, callback) => {
|
379 | if (node.compiler.watching) {
|
380 | node.compiler.watching.close(callback);
|
381 | } else {
|
382 | callback();
|
383 | }
|
384 | },
|
385 | () => callback(err)
|
386 | );
|
387 | }
|
388 | node.result = stats;
|
389 | running--;
|
390 | if (node.state === "running") {
|
391 | node.state = "done";
|
392 | for (const child of node.children) {
|
393 | if (child.state === "blocked") queue.enqueue(child);
|
394 | }
|
395 | } else if (node.state === "running-outdated") {
|
396 | node.state = "blocked";
|
397 | queue.enqueue(node);
|
398 | }
|
399 | processQueue();
|
400 | };
|
401 | |
402 |
|
403 |
|
404 |
|
405 | const nodeInvalidFromParent = node => {
|
406 | if (node.state === "done") {
|
407 | node.state = "blocked";
|
408 | } else if (node.state === "running") {
|
409 | node.state = "running-outdated";
|
410 | }
|
411 | for (const child of node.children) {
|
412 | nodeInvalidFromParent(child);
|
413 | }
|
414 | };
|
415 | |
416 |
|
417 |
|
418 |
|
419 | const nodeInvalid = node => {
|
420 | if (node.state === "done") {
|
421 | node.state = "pending";
|
422 | } else if (node.state === "running") {
|
423 | node.state = "running-outdated";
|
424 | }
|
425 | for (const child of node.children) {
|
426 | nodeInvalidFromParent(child);
|
427 | }
|
428 | };
|
429 | |
430 |
|
431 |
|
432 |
|
433 | const nodeChange = node => {
|
434 | nodeInvalid(node);
|
435 | if (node.state === "pending") {
|
436 | node.state = "blocked";
|
437 | }
|
438 | if (node.state === "blocked") {
|
439 | queue.enqueue(node);
|
440 | processQueue();
|
441 | }
|
442 | };
|
443 |
|
444 | const setupResults = [];
|
445 | nodes.forEach((node, i) => {
|
446 | setupResults.push(
|
447 | setup(
|
448 | node.compiler,
|
449 | i,
|
450 | nodeDone.bind(null, node),
|
451 | () => node.state !== "starting" && node.state !== "running",
|
452 | () => nodeChange(node),
|
453 | () => nodeInvalid(node)
|
454 | )
|
455 | );
|
456 | });
|
457 | let processing = true;
|
458 | const processQueue = () => {
|
459 | if (processing) return;
|
460 | processing = true;
|
461 | process.nextTick(processQueueWorker);
|
462 | };
|
463 | const processQueueWorker = () => {
|
464 | while (running < parallelism && queue.length > 0 && !errored) {
|
465 | const node = queue.dequeue();
|
466 | if (
|
467 | node.state === "queued" ||
|
468 | (node.state === "blocked" &&
|
469 | node.parents.every(p => p.state === "done"))
|
470 | ) {
|
471 | running++;
|
472 | node.state = "starting";
|
473 | run(node.compiler, nodeDone.bind(null, node));
|
474 | node.state = "running";
|
475 | }
|
476 | }
|
477 | processing = false;
|
478 | if (
|
479 | !errored &&
|
480 | running === 0 &&
|
481 | nodes.every(node => node.state === "done")
|
482 | ) {
|
483 | const stats = [];
|
484 | for (const node of nodes) {
|
485 | const result = node.result;
|
486 | if (result) {
|
487 | node.result = undefined;
|
488 | stats.push(result);
|
489 | }
|
490 | }
|
491 | if (stats.length > 0) {
|
492 | callback(null, new MultiStats(stats));
|
493 | }
|
494 | }
|
495 | };
|
496 | processQueueWorker();
|
497 | return setupResults;
|
498 | }
|
499 |
|
500 | |
501 |
|
502 |
|
503 |
|
504 |
|
505 | watch(watchOptions, handler) {
|
506 | if (this.running) {
|
507 | return handler(new ConcurrentCompilationError());
|
508 | }
|
509 | this.running = true;
|
510 |
|
511 | if (this.validateDependencies(handler)) {
|
512 | const watchings = this._runGraph(
|
513 | (compiler, idx, callback, isBlocked, setChanged, setInvalid) => {
|
514 | const watching = compiler.watch(
|
515 | Array.isArray(watchOptions) ? watchOptions[idx] : watchOptions,
|
516 | callback
|
517 | );
|
518 | if (watching) {
|
519 | watching._onInvalid = setInvalid;
|
520 | watching._onChange = setChanged;
|
521 | watching._isBlocked = isBlocked;
|
522 | }
|
523 | return watching;
|
524 | },
|
525 | (compiler, initial, callback) => {
|
526 | if (!compiler.watching.running) compiler.watching.invalidate();
|
527 | },
|
528 | handler
|
529 | );
|
530 | return new MultiWatching(watchings, this);
|
531 | }
|
532 |
|
533 | return new MultiWatching([], this);
|
534 | }
|
535 |
|
536 | |
537 |
|
538 |
|
539 |
|
540 | run(callback) {
|
541 | if (this.running) {
|
542 | return callback(new ConcurrentCompilationError());
|
543 | }
|
544 | this.running = true;
|
545 |
|
546 | if (this.validateDependencies(callback)) {
|
547 | this._runGraph(
|
548 | () => {},
|
549 | (compiler, callback) => compiler.run(callback),
|
550 | (err, stats) => {
|
551 | this.running = false;
|
552 |
|
553 | if (callback !== undefined) {
|
554 | return callback(err, stats);
|
555 | }
|
556 | }
|
557 | );
|
558 | }
|
559 | }
|
560 |
|
561 | purgeInputFileSystem() {
|
562 | for (const compiler of this.compilers) {
|
563 | if (compiler.inputFileSystem && compiler.inputFileSystem.purge) {
|
564 | compiler.inputFileSystem.purge();
|
565 | }
|
566 | }
|
567 | }
|
568 |
|
569 | |
570 |
|
571 |
|
572 |
|
573 | close(callback) {
|
574 | asyncLib.each(
|
575 | this.compilers,
|
576 | (compiler, callback) => {
|
577 | compiler.close(callback);
|
578 | },
|
579 | callback
|
580 | );
|
581 | }
|
582 | };
|