1 | 'use strict';
|
2 |
|
3 | const Hoek = require('@hapi/hoek');
|
4 | const Seedrandom = require('seedrandom');
|
5 | const WillCall = require('will-call');
|
6 |
|
7 | const Reporters = require('./reporters');
|
8 | const Coverage = require('./coverage');
|
9 | const Linters = require('./lint');
|
10 | const Leaks = require('./leaks');
|
11 | const Types = require('./types');
|
12 | const Utils = require('./utils');
|
13 |
|
14 |
|
15 | const internals = {};
|
16 |
|
17 |
|
18 |
|
19 |
|
20 | const Date = global.Date;
|
21 | const setTimeout = global.setTimeout;
|
22 | const clearTimeout = global.clearTimeout;
|
23 | const setImmediate = global.setImmediate;
|
24 |
|
25 |
|
26 | Error.stackTraceLimit = Infinity;
|
27 |
|
28 |
|
29 | internals.defaults = {
|
30 | assert: null,
|
31 | bail: false,
|
32 | coverage: false,
|
33 |
|
34 |
|
35 |
|
36 | colors: null,
|
37 | dry: false,
|
38 | environment: 'test',
|
39 |
|
40 |
|
41 | grep: null,
|
42 | ids: [],
|
43 | globals: null,
|
44 | leaks: true,
|
45 | timeout: 2000,
|
46 | output: process.stdout,
|
47 | progress: 1,
|
48 | reporter: 'console',
|
49 | shuffle: false,
|
50 | seed: Math.floor(Math.random() * 1000),
|
51 |
|
52 |
|
53 | threshold: 0,
|
54 |
|
55 | lint: false,
|
56 | 'lint-fix': false,
|
57 | 'lint-errors-threshold': 0,
|
58 | 'lint-warnings-threshold': 0
|
59 | };
|
60 |
|
61 |
|
62 | exports.report = async (scripts, options) => {
|
63 |
|
64 | const settings = Utils.mergeOptions(internals.defaults, options);
|
65 | settings.environment = settings.environment.trim();
|
66 | const reporter = Reporters.generate(settings);
|
67 |
|
68 | const executeScripts = async function () {
|
69 |
|
70 | try {
|
71 | const result = await exports.execute(scripts, settings, reporter);
|
72 |
|
73 | if (settings.leaks) {
|
74 | result.leaks = Leaks.detect(settings.globals);
|
75 | }
|
76 |
|
77 | if (settings.coverage) {
|
78 | result.coverage = await Coverage.analyze(settings);
|
79 | }
|
80 |
|
81 | if (settings.types) {
|
82 | result.types = await Types.validate(settings);
|
83 | }
|
84 |
|
85 | if (settings.shuffle) {
|
86 | result.seed = settings.seed;
|
87 | result.shuffle = true;
|
88 | }
|
89 |
|
90 | return result;
|
91 | }
|
92 | catch (ex) {
|
93 |
|
94 |
|
95 | const outputStream = [].concat(options.output).find((output) => !!output.write);
|
96 | if (outputStream) {
|
97 | outputStream.write(ex.toString() + '\n');
|
98 | }
|
99 | else {
|
100 | console.error(ex.toString());
|
101 | }
|
102 |
|
103 | return process.exit(1);
|
104 |
|
105 | }
|
106 | };
|
107 |
|
108 | const executeLint = async function () {
|
109 |
|
110 | return settings.lint && await Linters.lint(settings);
|
111 | };
|
112 |
|
113 | const results = await Promise.all([executeScripts(), executeLint()]);
|
114 | const notebook = results[0];
|
115 | notebook.lint = results[1];
|
116 |
|
117 | if (settings.assert) {
|
118 | notebook.assertions = settings.assert.count && settings.assert.count();
|
119 | const incompletes = settings.assert.incomplete && settings.assert.incomplete();
|
120 | if (incompletes) {
|
121 | for (let i = 0; i < incompletes.length; ++i) {
|
122 | const error = new Error('Incomplete assertion at ' + incompletes[i]);
|
123 | error.stack = undefined;
|
124 | notebook.errors.push(error);
|
125 | }
|
126 | }
|
127 | }
|
128 |
|
129 | return reporter.finalize(notebook);
|
130 | };
|
131 |
|
132 |
|
133 | exports.execute = async function (scripts, options, reporter) {
|
134 |
|
135 | const settings = Utils.mergeOptions(internals.defaults, options);
|
136 |
|
137 | scripts = [].concat(scripts);
|
138 |
|
139 | if (settings.shuffle) {
|
140 | internals.shuffle(scripts, settings.seed);
|
141 | }
|
142 |
|
143 | const experiments = scripts.map((script) => {
|
144 |
|
145 | script._executed = true;
|
146 | return script._root;
|
147 | });
|
148 |
|
149 | const onlyNodes = Hoek.flatten(scripts.map((script) => script._onlyNodes));
|
150 | if (onlyNodes.length) {
|
151 | onlyNodes.forEach((onlyNode) => {
|
152 |
|
153 | internals.markParentsAsOnly(onlyNode.experiment);
|
154 | });
|
155 |
|
156 | onlyNodes.forEach((onlyNode) => {
|
157 |
|
158 | internals.skipAllButOnly(scripts, onlyNode);
|
159 | });
|
160 | }
|
161 |
|
162 | reporter = reporter || { test: function () { }, start: function () { } };
|
163 |
|
164 | if (settings.environment) {
|
165 | process.env.NODE_ENV = settings.environment;
|
166 | }
|
167 |
|
168 | const filters = {
|
169 | ids: settings.ids,
|
170 | grep: settings.grep ? new RegExp(settings.grep) : null
|
171 | };
|
172 |
|
173 | const count = internals.count(experiments, { filters });
|
174 | reporter.start({ count });
|
175 |
|
176 | const startTime = Date.now();
|
177 | const state = {
|
178 | report: {
|
179 | tests: [],
|
180 | failures: 0,
|
181 | errors: []
|
182 | },
|
183 | reporter,
|
184 | filters,
|
185 | options: settings,
|
186 | only: onlyNodes
|
187 | };
|
188 |
|
189 | await internals.executeExperiments(experiments, state, settings.dry);
|
190 | const notebook = {
|
191 | ms: Date.now() - startTime,
|
192 | tests: state.report.tests,
|
193 | failures: state.report.failures,
|
194 | errors: state.report.errors
|
195 | };
|
196 |
|
197 | return notebook;
|
198 | };
|
199 |
|
200 |
|
201 | internals.skipAllButOnly = (scripts, onlyNode) => {
|
202 |
|
203 | let currentExperiment = onlyNode.experiment;
|
204 | if (onlyNode.test) {
|
205 | currentExperiment.tests
|
206 | .filter(internals.not(onlyNode.test))
|
207 | .filter(internals.notOnly)
|
208 | .forEach(internals.enableSkip);
|
209 |
|
210 | currentExperiment.experiments
|
211 | .filter(internals.notOnly)
|
212 | .forEach(internals.enableSkip);
|
213 | }
|
214 |
|
215 | while (currentExperiment.parent) {
|
216 | currentExperiment.parent.tests
|
217 | .filter(internals.notOnly)
|
218 | .forEach(internals.enableSkip);
|
219 |
|
220 | currentExperiment.parent.experiments
|
221 | .filter(internals.not(currentExperiment))
|
222 | .filter(internals.notOnly)
|
223 | .filter((experiment) => {
|
224 |
|
225 | return experiment.tests.every(internals.notOnly);
|
226 | })
|
227 | .forEach(internals.enableSkip);
|
228 |
|
229 | currentExperiment = currentExperiment.parent;
|
230 | }
|
231 |
|
232 | scripts.forEach((script) => {
|
233 |
|
234 | if (!script._onlyNodes.length) {
|
235 | internals.enableSkip(script._root);
|
236 | }
|
237 | });
|
238 | };
|
239 |
|
240 | internals.not = (excludedElement) => {
|
241 |
|
242 | return (element) => element !== excludedElement;
|
243 | };
|
244 |
|
245 | internals.notOnly = (element) => {
|
246 |
|
247 | return !element.options.only;
|
248 | };
|
249 |
|
250 | internals.enableSkip = (element) => {
|
251 |
|
252 | element.options.skip = true;
|
253 | };
|
254 |
|
255 | internals.markParentsAsOnly = (element) => {
|
256 |
|
257 | if (!element.options.skip) {
|
258 |
|
259 | element.options.only = true;
|
260 |
|
261 | if (element.parent) {
|
262 |
|
263 | internals.markParentsAsOnly(element.parent);
|
264 | }
|
265 | }
|
266 | };
|
267 |
|
268 | internals.shuffle = function (scripts, seed) {
|
269 |
|
270 | const random = Seedrandom(seed);
|
271 |
|
272 | const last = scripts.length - 1;
|
273 | for (let i = 0; i < scripts.length; ++i) {
|
274 | const rand = i + Math.floor(random() * (last - i + 1));
|
275 | const temp = scripts[i];
|
276 | scripts[i] = scripts[rand];
|
277 | scripts[rand] = temp;
|
278 | }
|
279 | };
|
280 |
|
281 |
|
282 | internals.executeExperiments = async function (experiments, state, skip, parentContext) {
|
283 |
|
284 | for (const experiment of experiments) {
|
285 | const skipExperiment = skip || experiment.options.skip || !internals.experimentHasTests(experiment, state) || (state.options.bail && state.report.failures);
|
286 |
|
287 | state.currentContext = parentContext ? Hoek.clone(parentContext) : {};
|
288 |
|
289 |
|
290 |
|
291 | if (!skipExperiment) {
|
292 | try {
|
293 | await internals.executeDeps(experiment.befores, state);
|
294 | }
|
295 | catch (ex) {
|
296 | internals.fail([experiment], state, skip, '\'before\' action failed');
|
297 | state.report.errors.push(ex);
|
298 |
|
299 |
|
300 | continue;
|
301 | }
|
302 | }
|
303 |
|
304 |
|
305 |
|
306 | await internals.executeTests(experiment, state, skipExperiment);
|
307 |
|
308 |
|
309 |
|
310 | await internals.executeExperiments(experiment.experiments, state, skipExperiment, state.currentContext);
|
311 |
|
312 |
|
313 |
|
314 | if (!skipExperiment) {
|
315 | try {
|
316 | await internals.executeDeps(experiment.afters, state);
|
317 | }
|
318 | catch (ex) {
|
319 | internals.fail([experiment], state, skip, '\'after\' action failed');
|
320 | state.report.errors.push(ex);
|
321 | }
|
322 | }
|
323 | }
|
324 | };
|
325 |
|
326 |
|
327 | internals.executeDeps = async function (deps, state) {
|
328 |
|
329 | if (!deps || !deps.length) {
|
330 | return;
|
331 | }
|
332 |
|
333 | for (const dep of deps) {
|
334 | dep.options.timeout = Number.isSafeInteger(dep.options.timeout) ? dep.options.timeout : state.options['context-timeout'];
|
335 | await internals.protect(dep, state);
|
336 | }
|
337 | };
|
338 |
|
339 |
|
340 | internals.executeTests = async function (experiment, state, skip) {
|
341 |
|
342 | if (!experiment.tests.length) {
|
343 | return;
|
344 | }
|
345 |
|
346 |
|
347 |
|
348 | const befores = skip ? [] : internals.collectDeps(experiment, 'beforeEaches');
|
349 | const afters = skip ? [] : internals.collectDeps(experiment, 'afterEaches');
|
350 |
|
351 |
|
352 |
|
353 | const execute = async function (test) {
|
354 |
|
355 | const isNotFiltered = state.filters.ids.length && !state.filters.ids.includes(test.id);
|
356 | const isNotGrepped = state.filters.grep && !state.filters.grep.test(test.title);
|
357 |
|
358 | if (isNotFiltered || isNotGrepped) {
|
359 | return new Promise((resolve) => {
|
360 |
|
361 | setImmediate(resolve);
|
362 | });
|
363 | }
|
364 |
|
365 | const isSkipped = skip || test.options.skip || (state.options.bail && state.report.failures);
|
366 |
|
367 | if (!test.fn ||
|
368 | isSkipped) {
|
369 |
|
370 | test[test.fn ? 'skipped' : 'todo'] = true;
|
371 | test.duration = 0;
|
372 | state.report.tests.push(test);
|
373 | state.reporter.test(test);
|
374 | return new Promise((resolve) => {
|
375 |
|
376 | setImmediate(resolve);
|
377 | });
|
378 | }
|
379 |
|
380 |
|
381 |
|
382 | try {
|
383 | await internals.executeDeps(befores, state);
|
384 | }
|
385 | catch (ex) {
|
386 | internals.failTest(test, state, skip, ex);
|
387 | state.report.errors.push(ex);
|
388 | return;
|
389 | }
|
390 |
|
391 |
|
392 |
|
393 | const start = Date.now();
|
394 | try {
|
395 | test.context = Hoek.clone(state.currentContext);
|
396 | await internals.protect(test, state);
|
397 | }
|
398 | catch (ex) {
|
399 | state.report.failures++;
|
400 | test.err = ex;
|
401 | test.timeout = ex.timeout;
|
402 | }
|
403 |
|
404 | test.duration = Date.now() - start;
|
405 |
|
406 | state.report.tests.push(test);
|
407 | state.reporter.test(test);
|
408 |
|
409 |
|
410 |
|
411 | try {
|
412 | await internals.executeDeps(afters, state);
|
413 | }
|
414 | catch (ex) {
|
415 | state.report.failures++;
|
416 | state.report.errors.push(ex);
|
417 | }
|
418 |
|
419 | return;
|
420 | };
|
421 |
|
422 | for (const test of experiment.tests) {
|
423 | await execute(test);
|
424 | }
|
425 |
|
426 | return;
|
427 | };
|
428 |
|
429 |
|
430 | internals.experimentHasTests = function (experiment, state) {
|
431 |
|
432 | if (experiment.experiments.length) {
|
433 | const experimentsHasTests = experiment.experiments.some((childExperiment) => {
|
434 |
|
435 | return internals.experimentHasTests(childExperiment, state);
|
436 | });
|
437 |
|
438 | if (experimentsHasTests) {
|
439 | return true;
|
440 | }
|
441 | }
|
442 |
|
443 | const hasTests = experiment.tests.some((test) => {
|
444 |
|
445 | if ((state.filters.ids.length && state.filters.ids.indexOf(test.id) === -1) ||
|
446 | (state.filters.grep && !state.filters.grep.test(test.title))) {
|
447 |
|
448 | return false;
|
449 | }
|
450 |
|
451 | if (!test.options.skip && test.fn) {
|
452 | return true;
|
453 | }
|
454 | });
|
455 |
|
456 | return hasTests;
|
457 | };
|
458 |
|
459 |
|
460 | internals.collectDeps = function (experiment, key) {
|
461 |
|
462 | const set = [];
|
463 |
|
464 | // if we are looking at afterEaches, we want to run our parent's blocks before ours (unshift onto front)
|
465 | const arrayAddFn = key === 'afterEaches' ? Array.prototype.unshift : Array.prototype.push;
|
466 |
|
467 | if (experiment.parent) {
|
468 | arrayAddFn.apply(set, internals.collectDeps(experiment.parent, key));
|
469 | }
|
470 |
|
471 | arrayAddFn.apply(set, experiment[key] || []);
|
472 | return set;
|
473 | };
|
474 |
|
475 |
|
476 | internals.createItemTimeout = (item, ms, finish) => {
|
477 |
|
478 | return setTimeout(() => {
|
479 |
|
480 | const error = new Error(`Timed out (${ms}ms) - ${item.title}`);
|
481 | error.timeout = true;
|
482 | finish(error, 'timeout');
|
483 | }, ms);
|
484 | };
|
485 |
|
486 |
|
487 | internals.protect = function (item, state) {
|
488 |
|
489 | let isFirst = true;
|
490 | let timeoutId;
|
491 | let countBefore = -1;
|
492 | let failedWithUnhandledRejection = false;
|
493 | let failedWithUncaughtException = false;
|
494 |
|
495 | if (state.options.assert && state.options.assert.count) {
|
496 | countBefore = state.options.assert.count();
|
497 | }
|
498 |
|
499 | return new Promise((resolve, reject) => {
|
500 |
|
501 | const flags = { notes: [], context: item.context || state.currentContext };
|
502 | flags.note = (note) => {
|
503 |
|
504 | flags.notes.push(note);
|
505 | };
|
506 |
|
507 | const willcall = new WillCall();
|
508 | flags.mustCall = willcall.expect.bind(willcall);
|
509 |
|
510 | const finish = async function (err, cause) {
|
511 |
|
512 | clearTimeout(timeoutId);
|
513 | timeoutId = null;
|
514 |
|
515 | item.notes = (item.notes || []).concat(flags.notes);
|
516 |
|
517 | process.removeListener('unhandledRejection', promiseRejectionHandler);
|
518 | process.removeListener('uncaughtException', processUncaughtExceptionHandler);
|
519 |
|
520 | if (cause && (err instanceof Error === false)) {
|
521 | const data = err;
|
522 | err = new Error(`Non Error object received or caught (${cause})`);
|
523 | err.data = data;
|
524 | }
|
525 |
|
526 |
|
527 |
|
528 | if (failedWithUnhandledRejection || failedWithUncaughtException) {
|
529 | return reject(err);
|
530 | }
|
531 |
|
532 |
|
533 | if (state.options.assert && state.options.assert.count) {
|
534 | item.assertions = state.options.assert.count() - countBefore;
|
535 | }
|
536 |
|
537 | if (flags.onCleanup) {
|
538 |
|
539 |
|
540 | const onCleanupError = (err) => {
|
541 |
|
542 | return reject(err);
|
543 | };
|
544 |
|
545 |
|
546 | process.once('uncaughtException', onCleanupError);
|
547 |
|
548 | try {
|
549 | await flags.onCleanup();
|
550 | }
|
551 | catch (ex) {
|
552 | return reject(ex);
|
553 | }
|
554 |
|
555 | process.removeListener('uncaughtException', onCleanupError);
|
556 | }
|
557 |
|
558 | if (item.options.plan !== undefined) {
|
559 | if (item.options.plan !== item.assertions) {
|
560 | const planMessage = (item.assertions === undefined)
|
561 | ? `Expected ${item.options.plan} assertions, but no assertion library found`
|
562 | : `Expected ${item.options.plan} assertions, but found ${item.assertions}`;
|
563 | if (err && !/^Expected (at least )?\d+ assertions/.test(err.message)) {
|
564 | err.message = planMessage + ': ' + err.message;
|
565 | }
|
566 | else {
|
567 | err = new Error(planMessage);
|
568 | }
|
569 | }
|
570 | }
|
571 | else if (item.path &&
|
572 | state.options['default-plan-threshold'] &&
|
573 | (item.assertions === undefined ||
|
574 | item.assertions < state.options['default-plan-threshold'])) {
|
575 | const planMessage = (item.assertions === undefined)
|
576 | ? `Expected at least ${state.options['default-plan-threshold']} assertions, but no assertion library found`
|
577 | : `Expected at least ${state.options['default-plan-threshold']} assertions, but found ${item.assertions}`;
|
578 | if (err && !/^Expected (at least )?\d+ assertions/.test(err.message)) {
|
579 | err.message = planMessage + ': ' + err.message;
|
580 | }
|
581 | else {
|
582 | err = new Error(planMessage);
|
583 | }
|
584 | }
|
585 |
|
586 | if (!isFirst) {
|
587 | const message = `Thrown error received in test "${item.title}" (${cause})`;
|
588 | err = new Error(message);
|
589 | }
|
590 |
|
591 | isFirst = false;
|
592 |
|
593 | const callResults = willcall.check();
|
594 | if (callResults.length) {
|
595 | const callResult = callResults[0];
|
596 | err = new Error(`Expected ${callResult.name} to be executed ${callResult.expected}` +
|
597 | `time(s) but was executed ${callResult.actual} time(s)`);
|
598 | err.stack = callResult.stack;
|
599 | }
|
600 |
|
601 | item.context = null;
|
602 |
|
603 | return err ? reject(err) : resolve();
|
604 | };
|
605 |
|
606 | const ms = item.options.timeout !== undefined ? item.options.timeout : state.options.timeout;
|
607 |
|
608 |
|
609 |
|
610 | const promiseRejectionHandler = function (err) {
|
611 |
|
612 | if (flags.onUnhandledRejection) {
|
613 | flags.onUnhandledRejection(err);
|
614 | return;
|
615 | }
|
616 |
|
617 | failedWithUnhandledRejection = true;
|
618 | finish(err, 'unhandledRejection');
|
619 | };
|
620 |
|
621 | process.on('unhandledRejection', promiseRejectionHandler);
|
622 |
|
623 | const processUncaughtExceptionHandler = function (err) {
|
624 |
|
625 | if (flags.onUncaughtException) {
|
626 | try {
|
627 | flags.onUncaughtException(err);
|
628 | return;
|
629 | }
|
630 | catch (errInErrorhandling) {
|
631 | err = errInErrorhandling;
|
632 | }
|
633 | }
|
634 |
|
635 | failedWithUncaughtException = true;
|
636 | finish(err, 'uncaughtException');
|
637 | };
|
638 |
|
639 |
|
640 | process.on('uncaughtException', processUncaughtExceptionHandler);
|
641 |
|
642 | setImmediate(async () => {
|
643 |
|
644 | if (ms) {
|
645 | timeoutId = internals.createItemTimeout(item, ms, finish);
|
646 | }
|
647 |
|
648 | try {
|
649 | await item.fn.call(null, flags);
|
650 | finish();
|
651 | }
|
652 | catch (ex) {
|
653 | return finish(ex, 'unit test');
|
654 | }
|
655 | });
|
656 | });
|
657 | };
|
658 |
|
659 |
|
660 | internals.count = function (experiments, state) {
|
661 |
|
662 | state.count = state.count || 0;
|
663 | state.seq = state.seq || 0;
|
664 |
|
665 | for (let i = 0; i < experiments.length; ++i) {
|
666 | const experiment = experiments[i];
|
667 |
|
668 | for (let j = 0; j < experiment.tests.length; ++j) {
|
669 | const test = experiment.tests[j];
|
670 | test.id = ++state.seq;
|
671 | state.count += (state.filters.ids.length && state.filters.ids.indexOf(test.id) === -1) || (state.filters.grep && !state.filters.grep.test(test.title)) ? 0 : 1;
|
672 | }
|
673 |
|
674 | internals.count(experiment.experiments, state);
|
675 | }
|
676 |
|
677 | return state.count;
|
678 | };
|
679 |
|
680 |
|
681 | internals.fail = function (experiments, state, skip, err) {
|
682 |
|
683 | for (let i = 0; i < experiments.length; ++i) {
|
684 | const experiment = experiments[i];
|
685 |
|
686 | for (let j = 0; j < experiment.tests.length; ++j) {
|
687 | internals.failTest(experiment.tests[j], state, skip, err);
|
688 | }
|
689 |
|
690 | internals.fail(experiment.experiments, state, skip || experiment.options.skip, err);
|
691 | }
|
692 | };
|
693 |
|
694 |
|
695 | internals.failTest = function (test, state, skip, err) {
|
696 |
|
697 | if (!test.fn ||
|
698 | skip ||
|
699 | test.options.skip) {
|
700 |
|
701 | test[test.fn ? 'skipped' : 'todo'] = true;
|
702 | }
|
703 | else {
|
704 | state.report.failures++;
|
705 | test.err = err;
|
706 | }
|
707 |
|
708 | test.duration = 0;
|
709 |
|
710 | state.report.tests.push(test);
|
711 | state.reporter.test(test);
|
712 | };
|