1 | 'use strict';
|
2 |
|
3 | const Hoek = require('@hapi/hoek');
|
4 | const Seedrandom = require('seedrandom');
|
5 | const WillCall = require('will-call');
|
6 |
|
7 | const Reporters = require('./reporters');
|
8 | const Coverage = require('./coverage');
|
9 | const Linters = require('./lint');
|
10 | const Leaks = require('./leaks');
|
11 | const Types = require('./types');
|
12 |
|
13 |
|
14 | const internals = {};
|
15 |
|
16 |
|
17 |
|
18 |
|
19 | const Date = global.Date;
|
20 | const setTimeout = global.setTimeout;
|
21 | const clearTimeout = global.clearTimeout;
|
22 | const setImmediate = global.setImmediate;
|
23 |
|
24 |
|
25 | Error.stackTraceLimit = Infinity;
|
26 |
|
27 |
|
28 | internals.defaults = {
|
29 | assert: null,
|
30 | bail: false,
|
31 | coverage: false,
|
32 |
|
33 |
|
34 |
|
35 | colors: null,
|
36 | dry: false,
|
37 | environment: 'test',
|
38 |
|
39 |
|
40 | grep: null,
|
41 | ids: [],
|
42 | globals: null,
|
43 | leaks: true,
|
44 | timeout: 2000,
|
45 | output: process.stdout,
|
46 | progress: 1,
|
47 | reporter: 'console',
|
48 | retries: 5,
|
49 | shuffle: false,
|
50 | seed: Math.floor(Math.random() * 1000),
|
51 |
|
52 |
|
53 | threshold: 0,
|
54 |
|
55 | lint: false,
|
56 | 'lint-fix': false,
|
57 | 'lint-errors-threshold': 0,
|
58 | 'lint-warnings-threshold': 0
|
59 | };
|
60 |
|
61 |
|
62 | exports.report = async function (scripts, options) {
|
63 |
|
64 | const settings = Object.assign({}, internals.defaults, options);
|
65 | settings.environment = settings.environment.trim();
|
66 | const reporter = Reporters.generate(settings);
|
67 |
|
68 | const executeScripts = async function () {
|
69 |
|
70 | try {
|
71 | const result = await exports.execute(scripts, settings, reporter);
|
72 |
|
73 | if (settings.leaks) {
|
74 | result.leaks = Leaks.detect(settings.globals);
|
75 | }
|
76 |
|
77 | if (settings.coverage) {
|
78 | result.coverage = await Coverage.analyze(settings);
|
79 | }
|
80 |
|
81 | if (settings.types) {
|
82 | result.types = await Types.validate(settings);
|
83 | }
|
84 |
|
85 | if (settings.shuffle) {
|
86 | result.seed = settings.seed;
|
87 | result.shuffle = true;
|
88 | }
|
89 |
|
90 | return result;
|
91 | }
|
92 | catch (ex) {
|
93 |
|
94 |
|
95 | const outputStream = [].concat(options.output).find((output) => !!output.write);
|
96 | if (outputStream) {
|
97 | outputStream.write(ex.toString() + '\n');
|
98 | }
|
99 | else {
|
100 | console.error(ex.toString());
|
101 | }
|
102 |
|
103 | return process.exit(1);
|
104 |
|
105 | }
|
106 | };
|
107 |
|
108 | const executeLint = async function () {
|
109 |
|
110 | return settings.lint && await Linters.lint(settings);
|
111 | };
|
112 |
|
113 | const results = await Promise.all([executeScripts(), executeLint()]);
|
114 | const notebook = results[0];
|
115 | notebook.lint = results[1];
|
116 |
|
117 | if (settings.assert) {
|
118 | notebook.assertions = settings.assert.count && settings.assert.count();
|
119 | const incompletes = settings.assert.incomplete && settings.assert.incomplete();
|
120 | if (incompletes) {
|
121 | for (const incomplete of incompletes) {
|
122 | const error = new Error('Incomplete assertion at ' + incomplete);
|
123 | error.stack = undefined;
|
124 | notebook.errors.push(error);
|
125 | }
|
126 | }
|
127 | }
|
128 |
|
129 | return reporter.finalize(notebook);
|
130 | };
|
131 |
|
132 |
|
133 | exports.execute = async function (scripts, options, reporter) {
|
134 |
|
135 | const settings = Object.assign({}, internals.defaults, options);
|
136 |
|
137 | scripts = [].concat(scripts);
|
138 |
|
139 | if (settings.shuffle) {
|
140 | internals.shuffle(scripts, settings.seed);
|
141 | }
|
142 |
|
143 | const experiments = scripts.map((script) => {
|
144 |
|
145 | script._executed = true;
|
146 | return script._root;
|
147 | });
|
148 |
|
149 | const onlyNodes = Hoek.flatten(scripts.map((script) => script._onlyNodes));
|
150 | if (onlyNodes.length) {
|
151 | onlyNodes.forEach((onlyNode) => {
|
152 |
|
153 | internals.markParentsAsOnly(onlyNode.experiment);
|
154 | });
|
155 |
|
156 | onlyNodes.forEach((onlyNode) => {
|
157 |
|
158 | internals.skipAllButOnly(scripts, onlyNode);
|
159 | });
|
160 | }
|
161 |
|
162 | reporter = reporter || { test: function () { }, start: function () { } };
|
163 |
|
164 | if (settings.environment) {
|
165 | process.env.NODE_ENV = settings.environment;
|
166 | }
|
167 |
|
168 | const filters = {
|
169 | ids: settings.ids,
|
170 | grep: settings.grep ? new RegExp(settings.grep) : null
|
171 | };
|
172 |
|
173 | const count = internals.count(experiments, { filters });
|
174 | reporter.start({ count });
|
175 |
|
176 | const startTime = Date.now();
|
177 | const state = {
|
178 | report: {
|
179 | tests: [],
|
180 | failures: 0,
|
181 | errors: []
|
182 | },
|
183 | reporter,
|
184 | filters,
|
185 | options: settings,
|
186 | only: onlyNodes
|
187 | };
|
188 |
|
189 | await internals.executeExperiments(experiments, state, settings.dry);
|
190 | const notebook = {
|
191 | ms: Date.now() - startTime,
|
192 | tests: state.report.tests,
|
193 | failures: state.report.failures,
|
194 | errors: state.report.errors
|
195 | };
|
196 |
|
197 | return notebook;
|
198 | };
|
199 |
|
200 |
|
201 | internals.skipAllButOnly = function (scripts, onlyNode) {
|
202 |
|
203 | let currentExperiment = onlyNode.experiment;
|
204 | if (onlyNode.test) {
|
205 | currentExperiment.tests
|
206 | .filter(internals.not(onlyNode.test))
|
207 | .filter(internals.notOnly)
|
208 | .forEach(internals.enableSkip);
|
209 |
|
210 | currentExperiment.experiments
|
211 | .filter(internals.notOnly)
|
212 | .forEach(internals.enableSkip);
|
213 | }
|
214 |
|
215 | while (currentExperiment.parent) {
|
216 | currentExperiment.parent.tests
|
217 | .filter(internals.notOnly)
|
218 | .forEach(internals.enableSkip);
|
219 |
|
220 | currentExperiment.parent.experiments
|
221 | .filter(internals.not(currentExperiment))
|
222 | .filter(internals.notOnly)
|
223 | .filter((experiment) => {
|
224 |
|
225 | return experiment.tests.every(internals.notOnly);
|
226 | })
|
227 | .forEach(internals.enableSkip);
|
228 |
|
229 | currentExperiment = currentExperiment.parent;
|
230 | }
|
231 |
|
232 | scripts.forEach((script) => {
|
233 |
|
234 | if (!script._onlyNodes.length) {
|
235 | internals.enableSkip(script._root);
|
236 | }
|
237 | });
|
238 | };
|
239 |
|
240 |
|
241 | internals.not = function (excludedElement) {
|
242 |
|
243 | return (element) => element !== excludedElement;
|
244 | };
|
245 |
|
246 |
|
247 | internals.notOnly = (element) => {
|
248 |
|
249 | return !element.options.only;
|
250 | };
|
251 |
|
252 |
|
253 | internals.enableSkip = function (element) {
|
254 |
|
255 | element.options.skip = true;
|
256 | };
|
257 |
|
258 |
|
259 | internals.markParentsAsOnly = function (element) {
|
260 |
|
261 | if (!element.options.skip) {
|
262 | element.options.only = true;
|
263 |
|
264 | if (element.parent) {
|
265 | internals.markParentsAsOnly(element.parent);
|
266 | }
|
267 | }
|
268 | };
|
269 |
|
270 |
|
271 | internals.shuffle = function (scripts, seed) {
|
272 |
|
273 | const random = Seedrandom(seed);
|
274 |
|
275 | const last = scripts.length - 1;
|
276 | for (let i = 0; i < scripts.length; ++i) {
|
277 | const rand = i + Math.floor(random() * (last - i + 1));
|
278 | const temp = scripts[i];
|
279 | scripts[i] = scripts[rand];
|
280 | scripts[rand] = temp;
|
281 | }
|
282 | };
|
283 |
|
284 |
|
285 | internals.executeExperiments = async function (experiments, state, skip, parentContext) {
|
286 |
|
287 | for (const experiment of experiments) {
|
288 | const skipExperiment = skip ||
|
289 | experiment.options.skip ||
|
290 | !internals.experimentHasTests(experiment, state) ||
|
291 | (state.options.bail && state.report.failures);
|
292 |
|
293 | state.currentContext = parentContext ? Hoek.clone(parentContext) : {};
|
294 |
|
295 |
|
296 |
|
297 | if (!skipExperiment) {
|
298 | try {
|
299 | await internals.executeDeps(experiment.befores, state);
|
300 | }
|
301 | catch (ex) {
|
302 | internals.fail([experiment], state, skip, '\'before\' action failed');
|
303 | state.report.errors.push(ex);
|
304 |
|
305 |
|
306 | continue;
|
307 | }
|
308 | }
|
309 |
|
310 |
|
311 |
|
312 | await internals.executeTests(experiment, state, skipExperiment);
|
313 |
|
314 |
|
315 |
|
316 | await internals.executeExperiments(experiment.experiments, state, skipExperiment, state.currentContext);
|
317 |
|
318 |
|
319 |
|
320 | if (!skipExperiment) {
|
321 | try {
|
322 | await internals.executeDeps(experiment.afters, state);
|
323 | }
|
324 | catch (ex) {
|
325 | internals.fail([experiment], state, skip, '\'after\' action failed');
|
326 | state.report.errors.push(ex);
|
327 | }
|
328 | }
|
329 | }
|
330 | };
|
331 |
|
332 |
|
333 | internals.executeDeps = async function (deps, state) {
|
334 |
|
335 | if (!deps ||
|
336 | !deps.length) {
|
337 |
|
338 | return;
|
339 | }
|
340 |
|
341 | for (const dep of deps) {
|
342 | dep.options.timeout = Number.isSafeInteger(dep.options.timeout) ? dep.options.timeout : state.options['context-timeout'];
|
343 | await internals.protect(dep, state);
|
344 | }
|
345 | };
|
346 |
|
347 |
|
348 | internals.executeTests = async function (experiment, state, skip) {
|
349 |
|
350 | if (!experiment.tests.length) {
|
351 | return;
|
352 | }
|
353 |
|
354 |
|
355 |
|
356 | const befores = skip ? [] : internals.collectDeps(experiment, 'beforeEaches');
|
357 | const afters = skip ? [] : internals.collectDeps(experiment, 'afterEaches');
|
358 |
|
359 |
|
360 |
|
361 | const execute = async function (test) {
|
362 |
|
363 | const isNotFiltered = state.filters.ids.length && !state.filters.ids.includes(test.id);
|
364 | const isNotGrepped = state.filters.grep && !state.filters.grep.test(test.title);
|
365 |
|
366 | if (isNotFiltered ||
|
367 | isNotGrepped) {
|
368 |
|
369 | return new Promise((resolve) => setImmediate(resolve));
|
370 | }
|
371 |
|
372 | const isSkipped = skip || test.options.skip || (state.options.bail && state.report.failures);
|
373 |
|
374 | if (!test.fn ||
|
375 | isSkipped) {
|
376 |
|
377 | test[test.fn ? 'skipped' : 'todo'] = true;
|
378 | test.duration = 0;
|
379 | state.report.tests.push(test);
|
380 | state.reporter.test(test);
|
381 | return new Promise((resolve) => setImmediate(resolve));
|
382 | }
|
383 |
|
384 |
|
385 |
|
386 | try {
|
387 | await internals.executeDeps(befores, state);
|
388 | }
|
389 | catch (ex) {
|
390 | internals.failTest(test, state, skip, ex);
|
391 | state.report.errors.push(ex);
|
392 | return;
|
393 | }
|
394 |
|
395 |
|
396 |
|
397 | const start = Date.now();
|
398 | try {
|
399 | test.context = Hoek.clone(state.currentContext);
|
400 | await internals.protect(test, state);
|
401 | }
|
402 | catch (ex) {
|
403 | state.report.failures++;
|
404 | test.err = ex;
|
405 | test.timeout = ex.timeout;
|
406 | }
|
407 |
|
408 | test.duration = Date.now() - start;
|
409 |
|
410 | state.report.tests.push(test);
|
411 | state.reporter.test(test);
|
412 |
|
413 |
|
414 |
|
415 | try {
|
416 | await internals.executeDeps(afters, state);
|
417 | }
|
418 | catch (ex) {
|
419 | state.report.failures++;
|
420 | state.report.errors.push(ex);
|
421 | }
|
422 |
|
423 | return;
|
424 | };
|
425 |
|
426 | for (const test of experiment.tests) {
|
427 | await execute(test);
|
428 | }
|
429 |
|
430 | return;
|
431 | };
|
432 |
|
433 |
|
434 | internals.experimentHasTests = function (experiment, state) {
|
435 |
|
436 | if (experiment.experiments.length) {
|
437 | const experimentsHasTests = experiment.experiments.some((childExperiment) => {
|
438 |
|
439 | return internals.experimentHasTests(childExperiment, state);
|
440 | });
|
441 |
|
442 | if (experimentsHasTests) {
|
443 | return true;
|
444 | }
|
445 | }
|
446 |
|
447 | const hasTests = experiment.tests.some((test) => {
|
448 |
|
449 | if ((state.filters.ids.length && state.filters.ids.indexOf(test.id) === -1) ||
|
450 | (state.filters.grep && !state.filters.grep.test(test.title))) {
|
451 |
|
452 | return false;
|
453 | }
|
454 |
|
455 | if (!test.options.skip &&
|
456 | test.fn) {
|
457 |
|
458 | return true;
|
459 | }
|
460 | });
|
461 |
|
462 | return hasTests;
|
463 | };
|
464 |
|
465 |
|
466 | internals.collectDeps = function (experiment, key) {
|
467 |
|
468 | const set = [];
|
469 |
|
470 | // if we are looking at afterEaches, we want to run our parent's blocks before ours (unshift onto front)
|
471 | const arrayAddFn = key === 'afterEaches' ? Array.prototype.unshift : Array.prototype.push;
|
472 |
|
473 | if (experiment.parent) {
|
474 | arrayAddFn.apply(set, internals.collectDeps(experiment.parent, key));
|
475 | }
|
476 |
|
477 | arrayAddFn.apply(set, experiment[key] || []);
|
478 | return set;
|
479 | };
|
480 |
|
481 |
|
482 | internals.createItemTimeout = (item, ms, finish) => {
|
483 |
|
484 | return setTimeout(() => {
|
485 |
|
486 | const error = new Error(`Timed out (${ms}ms) - ${item.title}`);
|
487 | error.timeout = true;
|
488 | finish(error, 'timeout');
|
489 | }, ms);
|
490 | };
|
491 |
|
492 |
|
493 | internals.protect = function (item, state) {
|
494 |
|
495 | let isFirst = true;
|
496 | let timeoutId;
|
497 | let countBefore = -1;
|
498 | let failedWithUnhandledRejection = false;
|
499 | let failedWithUncaughtException = false;
|
500 |
|
501 | if (state.options.assert &&
|
502 | state.options.assert.count) {
|
503 |
|
504 | countBefore = state.options.assert.count();
|
505 | }
|
506 |
|
507 | return new Promise((resolve, reject) => {
|
508 |
|
509 | const flags = { notes: [], context: item.context || state.currentContext };
|
510 | flags.note = (note) => flags.notes.push(note);
|
511 |
|
512 | const willcall = new WillCall();
|
513 | flags.mustCall = willcall.expect.bind(willcall);
|
514 |
|
515 | const finish = async function (err, cause) {
|
516 |
|
517 | clearTimeout(timeoutId);
|
518 | timeoutId = null;
|
519 |
|
520 | item.notes = (item.notes || []).concat(flags.notes);
|
521 |
|
522 | process.removeListener('unhandledRejection', promiseRejectionHandler);
|
523 | process.removeListener('uncaughtException', processUncaughtExceptionHandler);
|
524 |
|
525 | if (cause &&
|
526 | (err instanceof Error === false)) {
|
527 |
|
528 | const data = err;
|
529 | err = new Error(`Non Error object received or caught (${cause})`);
|
530 | err.data = data;
|
531 | }
|
532 |
|
533 |
|
534 |
|
535 | if (failedWithUnhandledRejection || failedWithUncaughtException) {
|
536 | return reject(err);
|
537 | }
|
538 |
|
539 |
|
540 | if (state.options.assert &&
|
541 | state.options.assert.count) {
|
542 |
|
543 | item.assertions = state.options.assert.count() - countBefore;
|
544 | }
|
545 |
|
546 | if (flags.onCleanup) {
|
547 |
|
548 |
|
549 | const onCleanupError = (err) => {
|
550 |
|
551 | return reject(err);
|
552 | };
|
553 |
|
554 |
|
555 | process.once('uncaughtException', onCleanupError);
|
556 |
|
557 | try {
|
558 | await flags.onCleanup();
|
559 | }
|
560 | catch (ex) {
|
561 | return reject(ex);
|
562 | }
|
563 |
|
564 | process.removeListener('uncaughtException', onCleanupError);
|
565 | }
|
566 |
|
567 | if (item.options.plan !== undefined) {
|
568 | if (item.options.plan !== item.assertions) {
|
569 | const planMessage = (item.assertions === undefined)
|
570 | ? `Expected ${item.options.plan} assertions, but no assertion library found`
|
571 | : `Expected ${item.options.plan} assertions, but found ${item.assertions}`;
|
572 | if (err && !/^Expected (at least )?\d+ assertions/.test(err.message)) {
|
573 | err.message = planMessage + ': ' + err.message;
|
574 | }
|
575 | else {
|
576 | err = new Error(planMessage);
|
577 | }
|
578 | }
|
579 | }
|
580 | else if (item.path &&
|
581 | state.options['default-plan-threshold'] &&
|
582 | (item.assertions === undefined ||
|
583 | item.assertions < state.options['default-plan-threshold'])) {
|
584 | const planMessage = (item.assertions === undefined)
|
585 | ? `Expected at least ${state.options['default-plan-threshold']} assertions, but no assertion library found`
|
586 | : `Expected at least ${state.options['default-plan-threshold']} assertions, but found ${item.assertions}`;
|
587 | if (err && !/^Expected (at least )?\d+ assertions/.test(err.message)) {
|
588 | err.message = planMessage + ': ' + err.message;
|
589 | }
|
590 | else {
|
591 | err = new Error(planMessage);
|
592 | }
|
593 | }
|
594 |
|
595 | if (!isFirst) {
|
596 | const message = `Thrown error received in test "${item.title}" (${cause})`;
|
597 | err = new Error(message);
|
598 | }
|
599 |
|
600 | isFirst = false;
|
601 |
|
602 | const callResults = willcall.check();
|
603 | if (callResults.length) {
|
604 | const callResult = callResults[0];
|
605 | err = new Error(`Expected ${callResult.name} to be executed ${callResult.expected} ` +
|
606 | `time(s) but was executed ${callResult.actual} time(s)`);
|
607 | err.stack = callResult.stack;
|
608 | }
|
609 |
|
610 | item.context = null;
|
611 |
|
612 | return err ? reject(err) : resolve();
|
613 | };
|
614 |
|
615 | const ms = item.options.timeout !== undefined ? item.options.timeout : state.options.timeout;
|
616 |
|
617 |
|
618 |
|
619 |
|
620 | const promiseRejectionHandler = function (err) {
|
621 |
|
622 | if (flags.onUnhandledRejection) {
|
623 | flags.onUnhandledRejection(err);
|
624 | return;
|
625 | }
|
626 |
|
627 | failedWithUnhandledRejection = true;
|
628 | finish(err, 'unhandledRejection');
|
629 | };
|
630 |
|
631 | process.on('unhandledRejection', promiseRejectionHandler);
|
632 |
|
633 | const processUncaughtExceptionHandler = function (err) {
|
634 |
|
635 | if (flags.onUncaughtException) {
|
636 | try {
|
637 | flags.onUncaughtException(err);
|
638 | return;
|
639 | }
|
640 | catch (errInErrorhandling) {
|
641 | err = errInErrorhandling;
|
642 | }
|
643 | }
|
644 |
|
645 | failedWithUncaughtException = true;
|
646 | finish(err, 'uncaughtException');
|
647 | };
|
648 |
|
649 |
|
650 | process.on('uncaughtException', processUncaughtExceptionHandler);
|
651 |
|
652 | setImmediate(async () => {
|
653 |
|
654 | if (ms) {
|
655 | timeoutId = internals.createItemTimeout(item, ms, finish);
|
656 | }
|
657 |
|
658 | item.tries = 0;
|
659 | let retries = !item.options.retry ? 1 : (item.options.retry === true ? state.options.retries : item.options.retry);
|
660 | while (retries--) {
|
661 | ++item.tries;
|
662 |
|
663 | try {
|
664 | await item.fn.call(null, flags);
|
665 | finish();
|
666 | }
|
667 | catch (ex) {
|
668 | if (!retries) {
|
669 | return finish(ex, 'unit test');
|
670 | }
|
671 | }
|
672 | }
|
673 | });
|
674 | });
|
675 | };
|
676 |
|
677 |
|
678 | internals.count = function (experiments, state) {
|
679 |
|
680 | state.count = state.count || 0;
|
681 | state.seq = state.seq || 0;
|
682 |
|
683 | for (const experiment of experiments) {
|
684 | for (const test of experiment.tests) {
|
685 | test.id = ++state.seq;
|
686 | state.count += (state.filters.ids.length && state.filters.ids.indexOf(test.id) === -1) || (state.filters.grep && !state.filters.grep.test(test.title)) ? 0 : 1;
|
687 | }
|
688 |
|
689 | internals.count(experiment.experiments, state);
|
690 | }
|
691 |
|
692 | return state.count;
|
693 | };
|
694 |
|
695 |
|
696 | internals.fail = function (experiments, state, skip, err) {
|
697 |
|
698 | for (const experiment of experiments) {
|
699 | for (const test of experiment.tests) {
|
700 | internals.failTest(test, state, skip, err);
|
701 | }
|
702 |
|
703 | internals.fail(experiment.experiments, state, skip || experiment.options.skip, err);
|
704 | }
|
705 | };
|
706 |
|
707 |
|
708 | internals.failTest = function (test, state, skip, err) {
|
709 |
|
710 | if (!test.fn ||
|
711 | skip ||
|
712 | test.options.skip) {
|
713 |
|
714 | test[test.fn ? 'skipped' : 'todo'] = true;
|
715 | }
|
716 | else {
|
717 | state.report.failures++;
|
718 | test.err = err;
|
719 | }
|
720 |
|
721 | test.duration = 0;
|
722 |
|
723 | state.report.tests.push(test);
|
724 | state.reporter.test(test);
|
725 | };
|