1 | 'use strict';
|
2 |
|
3 | const Hoek = require('@hapi/hoek');
|
4 | const Seedrandom = require('seedrandom');
|
5 | const WillCall = require('will-call');
|
6 |
|
7 | const Modules = require('./modules');
|
8 | const Reporters = require('./reporters');
|
9 |
|
10 |
|
11 | const internals = {};
|
12 |
|
13 |
|
14 |
|
15 |
|
16 | const Date = global.Date;
|
17 | const setTimeout = global.setTimeout;
|
18 | const clearTimeout = global.clearTimeout;
|
19 | const setImmediate = global.setImmediate;
|
20 |
|
21 |
|
22 | Error.stackTraceLimit = Infinity;
|
23 |
|
24 |
|
25 | internals.defaults = {
|
26 | assert: null,
|
27 | bail: false,
|
28 | coverage: false,
|
29 |
|
30 |
|
31 |
|
32 | colors: null,
|
33 | dry: false,
|
34 | environment: 'test',
|
35 |
|
36 |
|
37 | grep: null,
|
38 | ids: [],
|
39 | globals: null,
|
40 | leaks: true,
|
41 | timeout: 2000,
|
42 | output: process.stdout,
|
43 | progress: 1,
|
44 | reporter: 'console',
|
45 | retries: 5,
|
46 | shuffle: false,
|
47 | seed: Math.floor(Math.random() * 1000),
|
48 |
|
49 |
|
50 | threshold: 0,
|
51 |
|
52 | lint: false,
|
53 | 'lint-fix': false,
|
54 | 'lint-errors-threshold': 0,
|
55 | 'lint-warnings-threshold': 0
|
56 | };
|
57 |
|
58 |
|
59 | exports.report = async function (scripts, options) {
|
60 |
|
61 | const settings = Object.assign({}, internals.defaults, options);
|
62 | settings.environment = settings.environment.trim();
|
63 | const reporter = Reporters.generate(settings);
|
64 |
|
65 | const executeScripts = async function () {
|
66 |
|
67 | try {
|
68 | const result = await exports.execute(scripts, settings, reporter);
|
69 |
|
70 | if (settings.leaks) {
|
71 | result.leaks = Modules.leaks.detect(settings.globals);
|
72 | }
|
73 |
|
74 | if (settings.coverage) {
|
75 | result.coverage = await Modules.coverage.analyze(settings);
|
76 | }
|
77 |
|
78 | if (settings.types) {
|
79 | result.types = await Modules.types.validate(settings);
|
80 | }
|
81 |
|
82 | if (settings.shuffle) {
|
83 | result.seed = settings.seed;
|
84 | result.shuffle = true;
|
85 | }
|
86 |
|
87 | return result;
|
88 | }
|
89 | catch (ex) {
|
90 |
|
91 |
|
92 | const outputStream = [].concat(options.output).find((output) => !!output.write);
|
93 | if (outputStream) {
|
94 | outputStream.write(ex.toString() + '\n');
|
95 | }
|
96 | else {
|
97 | console.error(ex.toString());
|
98 | }
|
99 |
|
100 | return process.exit(1);
|
101 |
|
102 | }
|
103 | };
|
104 |
|
105 | const executeLint = async function () {
|
106 |
|
107 | return settings.lint && await Modules.lint.lint(settings);
|
108 | };
|
109 |
|
110 | const results = await Promise.all([executeScripts(), executeLint()]);
|
111 | const notebook = results[0];
|
112 | notebook.lint = results[1];
|
113 |
|
114 | if (settings.assert) {
|
115 | notebook.assertions = settings.assert.count && settings.assert.count();
|
116 | const incompletes = settings.assert.incomplete && settings.assert.incomplete();
|
117 | if (incompletes) {
|
118 | for (const incomplete of incompletes) {
|
119 | const error = new Error('Incomplete assertion at ' + incomplete);
|
120 | error.stack = undefined;
|
121 | notebook.errors.push(error);
|
122 | }
|
123 | }
|
124 | }
|
125 |
|
126 | return reporter.finalize(notebook);
|
127 | };
|
128 |
|
129 |
|
130 | exports.execute = async function (scripts, options, reporter) {
|
131 |
|
132 | const settings = Object.assign({}, internals.defaults, options);
|
133 |
|
134 | scripts = [].concat(scripts);
|
135 |
|
136 | if (settings.shuffle) {
|
137 | internals.shuffle(scripts, settings.seed);
|
138 | }
|
139 |
|
140 | const experiments = scripts.map((script) => {
|
141 |
|
142 | script._executed = true;
|
143 | return script._root;
|
144 | });
|
145 |
|
146 | const onlyNodes = Hoek.flatten(scripts.map((script) => script._onlyNodes));
|
147 | if (onlyNodes.length) {
|
148 | onlyNodes.forEach((onlyNode) => {
|
149 |
|
150 | internals.markParentsAsOnly(onlyNode.experiment);
|
151 | });
|
152 |
|
153 | onlyNodes.forEach((onlyNode) => {
|
154 |
|
155 | internals.skipAllButOnly(scripts, onlyNode);
|
156 | });
|
157 | }
|
158 |
|
159 | reporter = reporter || { test: function () { }, start: function () { } };
|
160 |
|
161 | if (settings.environment) {
|
162 | process.env.NODE_ENV = settings.environment;
|
163 | }
|
164 |
|
165 | const filters = {
|
166 | ids: settings.ids,
|
167 | grep: settings.grep ? new RegExp(settings.grep) : null
|
168 | };
|
169 |
|
170 | const count = internals.count(experiments, { filters });
|
171 | reporter.start({ count });
|
172 |
|
173 | const startTime = Date.now();
|
174 | const state = {
|
175 | report: {
|
176 | tests: [],
|
177 | failures: 0,
|
178 | errors: []
|
179 | },
|
180 | reporter,
|
181 | filters,
|
182 | options: settings,
|
183 | only: onlyNodes
|
184 | };
|
185 |
|
186 | await internals.executeExperiments(experiments, state, settings.dry);
|
187 | const notebook = {
|
188 | ms: Date.now() - startTime,
|
189 | tests: state.report.tests,
|
190 | failures: state.report.failures,
|
191 | errors: state.report.errors
|
192 | };
|
193 |
|
194 | return notebook;
|
195 | };
|
196 |
|
197 |
|
198 | internals.skipAllButOnly = function (scripts, onlyNode) {
|
199 |
|
200 | let currentExperiment = onlyNode.experiment;
|
201 | if (onlyNode.test) {
|
202 | currentExperiment.tests
|
203 | .filter(internals.not(onlyNode.test))
|
204 | .filter(internals.notOnly)
|
205 | .forEach(internals.enableSkip);
|
206 |
|
207 | currentExperiment.experiments
|
208 | .filter(internals.notOnly)
|
209 | .forEach(internals.enableSkip);
|
210 | }
|
211 |
|
212 | while (currentExperiment.parent) {
|
213 | currentExperiment.parent.tests
|
214 | .filter(internals.notOnly)
|
215 | .forEach(internals.enableSkip);
|
216 |
|
217 | currentExperiment.parent.experiments
|
218 | .filter(internals.not(currentExperiment))
|
219 | .filter(internals.notOnly)
|
220 | .filter((experiment) => {
|
221 |
|
222 | return experiment.tests.every(internals.notOnly);
|
223 | })
|
224 | .forEach(internals.enableSkip);
|
225 |
|
226 | currentExperiment = currentExperiment.parent;
|
227 | }
|
228 |
|
229 | scripts.forEach((script) => {
|
230 |
|
231 | if (!script._onlyNodes.length) {
|
232 | internals.enableSkip(script._root);
|
233 | }
|
234 | });
|
235 | };
|
236 |
|
237 |
|
238 | internals.not = function (excludedElement) {
|
239 |
|
240 | return (element) => element !== excludedElement;
|
241 | };
|
242 |
|
243 |
|
244 | internals.notOnly = (element) => {
|
245 |
|
246 | return !element.options.only;
|
247 | };
|
248 |
|
249 |
|
250 | internals.enableSkip = function (element) {
|
251 |
|
252 | element.options.skip = true;
|
253 | };
|
254 |
|
255 |
|
256 | internals.markParentsAsOnly = function (element) {
|
257 |
|
258 | if (!element.options.skip) {
|
259 | element.options.only = true;
|
260 |
|
261 | if (element.parent) {
|
262 | internals.markParentsAsOnly(element.parent);
|
263 | }
|
264 | }
|
265 | };
|
266 |
|
267 |
|
268 | internals.shuffle = function (scripts, seed) {
|
269 |
|
270 | const random = Seedrandom(seed);
|
271 |
|
272 | const last = scripts.length - 1;
|
273 | for (let i = 0; i < scripts.length; ++i) {
|
274 | const rand = i + Math.floor(random() * (last - i + 1));
|
275 | const temp = scripts[i];
|
276 | scripts[i] = scripts[rand];
|
277 | scripts[rand] = temp;
|
278 | }
|
279 | };
|
280 |
|
281 |
|
282 | internals.executeExperiments = async function (experiments, state, skip, parentContext) {
|
283 |
|
284 | for (const experiment of experiments) {
|
285 | const skipExperiment = skip ||
|
286 | experiment.options.skip ||
|
287 | !internals.experimentHasTests(experiment, state) ||
|
288 | (state.options.bail && state.report.failures);
|
289 |
|
290 | state.currentContext = parentContext ? Hoek.clone(parentContext, { shallow: true }) : {};
|
291 |
|
292 |
|
293 |
|
294 | if (!skipExperiment) {
|
295 | try {
|
296 | await internals.executeDeps(experiment.befores, state);
|
297 | }
|
298 | catch (ex) {
|
299 | internals.fail([experiment], state, skip, '\'before\' action failed');
|
300 | state.report.errors.push(ex);
|
301 |
|
302 |
|
303 | continue;
|
304 | }
|
305 | }
|
306 |
|
307 |
|
308 |
|
309 | await internals.executeTests(experiment, state, skipExperiment);
|
310 |
|
311 |
|
312 |
|
313 | await internals.executeExperiments(experiment.experiments, state, skipExperiment, state.currentContext);
|
314 |
|
315 |
|
316 |
|
317 | if (!skipExperiment) {
|
318 | try {
|
319 | await internals.executeDeps(experiment.afters, state);
|
320 | }
|
321 | catch (ex) {
|
322 | internals.fail([experiment], state, skip, '\'after\' action failed');
|
323 | state.report.errors.push(ex);
|
324 | }
|
325 | }
|
326 | }
|
327 | };
|
328 |
|
329 |
|
330 | internals.executeDeps = async function (deps, state) {
|
331 |
|
332 | if (!deps ||
|
333 | !deps.length) {
|
334 |
|
335 | return;
|
336 | }
|
337 |
|
338 | for (const dep of deps) {
|
339 | dep.options.timeout = Number.isSafeInteger(dep.options.timeout) ? dep.options.timeout : state.options['context-timeout'];
|
340 | await internals.protect(dep, state);
|
341 | }
|
342 | };
|
343 |
|
344 |
|
345 | internals.executeTests = async function (experiment, state, skip) {
|
346 |
|
347 | if (!experiment.tests.length) {
|
348 | return;
|
349 | }
|
350 |
|
351 |
|
352 |
|
353 | const befores = skip ? [] : internals.collectDeps(experiment, 'beforeEaches');
|
354 | const afters = skip ? [] : internals.collectDeps(experiment, 'afterEaches');
|
355 |
|
356 |
|
357 |
|
358 | const execute = async function (test) {
|
359 |
|
360 | const isNotFiltered = state.filters.ids.length && !state.filters.ids.includes(test.id);
|
361 | const isNotGrepped = state.filters.grep && !state.filters.grep.test(test.title);
|
362 |
|
363 | if (isNotFiltered ||
|
364 | isNotGrepped) {
|
365 |
|
366 | return new Promise((resolve) => setImmediate(resolve));
|
367 | }
|
368 |
|
369 | const isSkipped = skip || test.options.skip || (state.options.bail && state.report.failures);
|
370 |
|
371 | if (!test.fn ||
|
372 | isSkipped) {
|
373 |
|
374 | test[test.fn ? 'skipped' : 'todo'] = true;
|
375 | test.duration = 0;
|
376 | state.report.tests.push(test);
|
377 | state.reporter.test(test);
|
378 | return new Promise((resolve) => setImmediate(resolve));
|
379 | }
|
380 |
|
381 |
|
382 |
|
383 | try {
|
384 | await internals.executeDeps(befores, state);
|
385 | }
|
386 | catch (ex) {
|
387 | internals.failTest(test, state, skip, ex);
|
388 | state.report.errors.push(ex);
|
389 | return;
|
390 | }
|
391 |
|
392 |
|
393 |
|
394 | const start = Date.now();
|
395 | try {
|
396 | test.context = Hoek.clone(state.currentContext, { shallow: true });
|
397 | await internals.protect(test, state);
|
398 | }
|
399 | catch (ex) {
|
400 | state.report.failures++;
|
401 | test.err = ex;
|
402 | test.timeout = ex.timeout;
|
403 | }
|
404 |
|
405 | test.duration = Date.now() - start;
|
406 |
|
407 | state.report.tests.push(test);
|
408 | state.reporter.test(test);
|
409 |
|
410 |
|
411 |
|
412 | try {
|
413 | await internals.executeDeps(afters, state);
|
414 | }
|
415 | catch (ex) {
|
416 | state.report.failures++;
|
417 | state.report.errors.push(ex);
|
418 | }
|
419 |
|
420 | return;
|
421 | };
|
422 |
|
423 | for (const test of experiment.tests) {
|
424 | await execute(test);
|
425 | }
|
426 |
|
427 | return;
|
428 | };
|
429 |
|
430 |
|
431 | internals.experimentHasTests = function (experiment, state) {
|
432 |
|
433 | if (experiment.experiments.length) {
|
434 | const experimentsHasTests = experiment.experiments.some((childExperiment) => {
|
435 |
|
436 | return internals.experimentHasTests(childExperiment, state);
|
437 | });
|
438 |
|
439 | if (experimentsHasTests) {
|
440 | return true;
|
441 | }
|
442 | }
|
443 |
|
444 | const hasTests = experiment.tests.some((test) => {
|
445 |
|
446 | if ((state.filters.ids.length && state.filters.ids.indexOf(test.id) === -1) ||
|
447 | (state.filters.grep && !state.filters.grep.test(test.title))) {
|
448 |
|
449 | return false;
|
450 | }
|
451 |
|
452 | if (!test.options.skip &&
|
453 | test.fn) {
|
454 |
|
455 | return true;
|
456 | }
|
457 | });
|
458 |
|
459 | return hasTests;
|
460 | };
|
461 |
|
462 |
|
463 | internals.collectDeps = function (experiment, key) {
|
464 |
|
465 | const set = [];
|
466 |
|
467 | // if we are looking at afterEaches, we want to run our parent's blocks before ours (unshift onto front)
|
468 | const arrayAddFn = key === 'afterEaches' ? Array.prototype.unshift : Array.prototype.push;
|
469 |
|
470 | if (experiment.parent) {
|
471 | arrayAddFn.apply(set, internals.collectDeps(experiment.parent, key));
|
472 | }
|
473 |
|
474 | arrayAddFn.apply(set, experiment[key] || []);
|
475 | return set;
|
476 | };
|
477 |
|
478 |
|
479 | internals.createItemTimeout = (item, ms, finish) => {
|
480 |
|
481 | return setTimeout(() => {
|
482 |
|
483 | const error = new Error(`Timed out (${ms}ms) - ${item.title}`);
|
484 | error.timeout = true;
|
485 | finish(error, 'timeout');
|
486 | }, ms);
|
487 | };
|
488 |
|
489 |
|
490 | internals.protect = function (item, state) {
|
491 |
|
492 | let isFirst = true;
|
493 | let timeoutId;
|
494 | let countBefore = -1;
|
495 | let failedWithUnhandledRejection = false;
|
496 | let failedWithUncaughtException = false;
|
497 |
|
498 | if (state.options.assert &&
|
499 | state.options.assert.count) {
|
500 |
|
501 | countBefore = state.options.assert.count();
|
502 | }
|
503 |
|
504 | return new Promise((resolve, reject) => {
|
505 |
|
506 | const flags = { notes: [], context: item.context || state.currentContext };
|
507 | flags.note = (note) => flags.notes.push(note);
|
508 |
|
509 | const willcall = new WillCall();
|
510 | flags.mustCall = willcall.expect.bind(willcall);
|
511 |
|
512 | const finish = async function (err, cause) {
|
513 |
|
514 | clearTimeout(timeoutId);
|
515 | timeoutId = null;
|
516 |
|
517 | item.notes = (item.notes || []).concat(flags.notes);
|
518 |
|
519 | process.removeListener('unhandledRejection', promiseRejectionHandler);
|
520 | process.removeListener('uncaughtException', processUncaughtExceptionHandler);
|
521 |
|
522 | if (cause &&
|
523 | (err instanceof Error === false)) {
|
524 |
|
525 | const data = err;
|
526 | err = new Error(`Non Error object received or caught (${cause})`);
|
527 | err.data = data;
|
528 | }
|
529 |
|
530 |
|
531 |
|
532 | if (failedWithUnhandledRejection || failedWithUncaughtException) {
|
533 | return reject(err);
|
534 | }
|
535 |
|
536 |
|
537 | if (state.options.assert &&
|
538 | state.options.assert.count) {
|
539 |
|
540 | item.assertions = state.options.assert.count() - countBefore;
|
541 | }
|
542 |
|
543 | if (flags.onCleanup) {
|
544 |
|
545 |
|
546 | const onCleanupError = (err) => {
|
547 |
|
548 | return reject(err);
|
549 | };
|
550 |
|
551 |
|
552 | process.once('uncaughtException', onCleanupError);
|
553 |
|
554 | try {
|
555 | await flags.onCleanup();
|
556 | }
|
557 | catch (ex) {
|
558 | return reject(ex);
|
559 | }
|
560 |
|
561 | process.removeListener('uncaughtException', onCleanupError);
|
562 | }
|
563 |
|
564 | if (item.options.plan !== undefined) {
|
565 | if (item.options.plan !== item.assertions) {
|
566 | const planMessage = (item.assertions === undefined)
|
567 | ? `Expected ${item.options.plan} assertions, but no assertion library found`
|
568 | : `Expected ${item.options.plan} assertions, but found ${item.assertions}`;
|
569 | if (err && !/^Expected (at least )?\d+ assertions/.test(err.message)) {
|
570 | err.message = planMessage + ': ' + err.message;
|
571 | }
|
572 | else {
|
573 | err = new Error(planMessage);
|
574 | }
|
575 | }
|
576 | }
|
577 | else if (item.path &&
|
578 | state.options['default-plan-threshold'] &&
|
579 | (item.assertions === undefined ||
|
580 | item.assertions < state.options['default-plan-threshold'])) {
|
581 | const planMessage = (item.assertions === undefined)
|
582 | ? `Expected at least ${state.options['default-plan-threshold']} assertions, but no assertion library found`
|
583 | : `Expected at least ${state.options['default-plan-threshold']} assertions, but found ${item.assertions}`;
|
584 | if (err && !/^Expected (at least )?\d+ assertions/.test(err.message)) {
|
585 | err.message = planMessage + ': ' + err.message;
|
586 | }
|
587 | else {
|
588 | err = new Error(planMessage);
|
589 | }
|
590 | }
|
591 |
|
592 | if (!isFirst) {
|
593 | const message = `Thrown error received in test "${item.title}" (${cause})`;
|
594 | err = new Error(message);
|
595 | }
|
596 |
|
597 | isFirst = false;
|
598 |
|
599 | const callResults = willcall.check();
|
600 | if (callResults.length) {
|
601 | const callResult = callResults[0];
|
602 | err = new Error(`Expected ${callResult.name} to be executed ${callResult.expected} ` +
|
603 | `time(s) but was executed ${callResult.actual} time(s)`);
|
604 | err.stack = callResult.stack;
|
605 | }
|
606 |
|
607 | item.context = null;
|
608 |
|
609 | return err ? reject(err) : resolve();
|
610 | };
|
611 |
|
612 | const ms = item.options.timeout !== undefined ? item.options.timeout : state.options.timeout;
|
613 |
|
614 |
|
615 |
|
616 |
|
617 | const promiseRejectionHandler = function (err) {
|
618 |
|
619 | if (flags.onUnhandledRejection) {
|
620 | flags.onUnhandledRejection(err);
|
621 | return;
|
622 | }
|
623 |
|
624 | failedWithUnhandledRejection = true;
|
625 | finish(err, 'unhandledRejection');
|
626 | };
|
627 |
|
628 | process.on('unhandledRejection', promiseRejectionHandler);
|
629 |
|
630 | const processUncaughtExceptionHandler = function (err) {
|
631 |
|
632 | if (flags.onUncaughtException) {
|
633 | try {
|
634 | flags.onUncaughtException(err);
|
635 | return;
|
636 | }
|
637 | catch (errInErrorhandling) {
|
638 | err = errInErrorhandling;
|
639 | }
|
640 | }
|
641 |
|
642 | failedWithUncaughtException = true;
|
643 | finish(err, 'uncaughtException');
|
644 | };
|
645 |
|
646 |
|
647 | process.on('uncaughtException', processUncaughtExceptionHandler);
|
648 |
|
649 | setImmediate(async () => {
|
650 |
|
651 | if (ms) {
|
652 | timeoutId = internals.createItemTimeout(item, ms, finish);
|
653 | }
|
654 |
|
655 | item.tries = 0;
|
656 | let retries = !item.options.retry ? 1 : (item.options.retry === true ? state.options.retries : item.options.retry);
|
657 | let lastError;
|
658 | while (retries--) {
|
659 | ++item.tries;
|
660 |
|
661 | try {
|
662 | await item.fn.call(null, flags);
|
663 |
|
664 | await new Promise((next) => {
|
665 |
|
666 | setImmediate(next);
|
667 | });
|
668 | return finish();
|
669 | }
|
670 | catch (ex) {
|
671 | lastError = ex;
|
672 | }
|
673 | }
|
674 |
|
675 | return finish(lastError, 'unit test');
|
676 | });
|
677 | });
|
678 | };
|
679 |
|
680 |
|
681 | internals.count = function (experiments, state) {
|
682 |
|
683 | state.count = state.count || 0;
|
684 | state.seq = state.seq || 0;
|
685 |
|
686 | for (const experiment of experiments) {
|
687 | for (const test of experiment.tests) {
|
688 | test.id = ++state.seq;
|
689 | state.count += (state.filters.ids.length && state.filters.ids.indexOf(test.id) === -1) || (state.filters.grep && !state.filters.grep.test(test.title)) ? 0 : 1;
|
690 | }
|
691 |
|
692 | internals.count(experiment.experiments, state);
|
693 | }
|
694 |
|
695 | return state.count;
|
696 | };
|
697 |
|
698 |
|
699 | internals.fail = function (experiments, state, skip, err) {
|
700 |
|
701 | for (const experiment of experiments) {
|
702 | for (const test of experiment.tests) {
|
703 | internals.failTest(test, state, skip, err);
|
704 | }
|
705 |
|
706 | internals.fail(experiment.experiments, state, skip || experiment.options.skip, err);
|
707 | }
|
708 | };
|
709 |
|
710 |
|
711 | internals.failTest = function (test, state, skip, err) {
|
712 |
|
713 | if (!test.fn ||
|
714 | skip ||
|
715 | test.options.skip) {
|
716 |
|
717 | test[test.fn ? 'skipped' : 'todo'] = true;
|
718 | }
|
719 | else {
|
720 | state.report.failures++;
|
721 | test.err = err;
|
722 | }
|
723 |
|
724 | test.duration = 0;
|
725 |
|
726 | state.report.tests.push(test);
|
727 | state.reporter.test(test);
|
728 | };
|