1 | 'use strict';
|
2 |
|
3 |
|
4 |
|
5 | const Hoek = require('hoek');
|
6 | const Seedrandom = require('seedrandom');
|
7 | const WillCall = require('will-call');
|
8 | const Reporters = require('./reporters');
|
9 | const Coverage = require('./coverage');
|
10 | const Linters = require('./lint');
|
11 | const Leaks = require('./leaks');
|
12 | const Utils = require('./utils');
|
13 |
|
14 |
|
15 |
|
16 | const Date = global.Date;
|
17 | const setTimeout = global.setTimeout;
|
18 | const clearTimeout = global.clearTimeout;
|
19 | const setImmediate = global.setImmediate;
|
20 |
|
21 |
|
22 |
|
23 |
|
24 | const internals = {};
|
25 |
|
26 |
|
27 | Error.stackTraceLimit = Infinity;
|
28 |
|
29 |
|
30 | internals.defaults = {
|
31 | assert: null,
|
32 | bail: false,
|
33 | coverage: false,
|
34 |
|
35 |
|
36 |
|
37 | colors: null,
|
38 | dry: false,
|
39 | environment: 'test',
|
40 |
|
41 |
|
42 | grep: null,
|
43 | ids: [],
|
44 | globals: null,
|
45 | leaks: true,
|
46 | timeout: 2000,
|
47 | output: process.stdout,
|
48 | progress: 1,
|
49 | reporter: 'console',
|
50 | shuffle: false,
|
51 | seed: Math.floor(Math.random() * 1000),
|
52 |
|
53 |
|
54 | threshold: 0,
|
55 |
|
56 | lint: false,
|
57 | 'lint-fix': false,
|
58 | 'lint-errors-threshold': 0,
|
59 | 'lint-warnings-threshold': 0
|
60 | };
|
61 |
|
62 |
|
63 | exports.report = async (scripts, options) => {
|
64 |
|
65 | const settings = Utils.mergeOptions(internals.defaults, options);
|
66 | settings.environment = settings.environment.trim();
|
67 | const reporter = Reporters.generate(settings);
|
68 |
|
69 | const executeScripts = async function () {
|
70 |
|
71 | try {
|
72 | const result = await exports.execute(scripts, settings, reporter);
|
73 |
|
74 | if (settings.leaks) {
|
75 | result.leaks = Leaks.detect(settings.globals);
|
76 | }
|
77 |
|
78 | if (settings.coverage) {
|
79 | result.coverage = await Coverage.analyze(settings);
|
80 | }
|
81 |
|
82 | if (settings.shuffle) {
|
83 | result.seed = settings.seed;
|
84 | result.shuffle = true;
|
85 | }
|
86 |
|
87 | return Promise.resolve(result);
|
88 | }
|
89 | catch (ex) {
|
90 |
|
91 |
|
92 | const outputStream = [].concat(options.output).find((output) => !!output.write);
|
93 | if (outputStream) {
|
94 | outputStream.write(ex.toString() + '\n');
|
95 | }
|
96 | else {
|
97 | console.error(ex.toString());
|
98 | }
|
99 |
|
100 | return process.exit(1);
|
101 |
|
102 | }
|
103 | };
|
104 |
|
105 | const executeLint = async function () {
|
106 |
|
107 | return settings.lint ? await Linters.lint(settings) : Promise.resolve();
|
108 | };
|
109 |
|
110 | const results = await Promise.all([executeScripts(), executeLint()]);
|
111 | const notebook = results[0];
|
112 | notebook.lint = results[1];
|
113 |
|
114 | if (settings.assert) {
|
115 | notebook.assertions = settings.assert.count && settings.assert.count();
|
116 | const incompletes = settings.assert.incomplete && settings.assert.incomplete();
|
117 | if (incompletes) {
|
118 | for (let i = 0; i < incompletes.length; ++i) {
|
119 | const error = new Error('Incomplete assertion at ' + incompletes[i]);
|
120 | error.stack = undefined;
|
121 | notebook.errors.push(error);
|
122 | }
|
123 | }
|
124 | }
|
125 |
|
126 | return reporter.finalize(notebook);
|
127 | };
|
128 |
|
129 |
|
130 | exports.execute = async function (scripts, options, reporter) {
|
131 |
|
132 | const settings = Utils.mergeOptions(internals.defaults, options);
|
133 |
|
134 | scripts = [].concat(scripts);
|
135 |
|
136 | if (settings.shuffle) {
|
137 | internals.shuffle(scripts, settings.seed);
|
138 | }
|
139 |
|
140 | const experiments = scripts.map((script) => {
|
141 |
|
142 | script._executed = true;
|
143 | return script._root;
|
144 | });
|
145 |
|
146 | const onlyNodes = Hoek.flatten(scripts.map((script) => script._onlyNodes));
|
147 | if (onlyNodes.length) {
|
148 | onlyNodes.forEach((onlyNode) => {
|
149 |
|
150 | internals.skipAllButOnly(scripts, onlyNode);
|
151 | });
|
152 | }
|
153 |
|
154 | reporter = reporter || { test: function () { }, start: function () { } };
|
155 |
|
156 | if (settings.environment) {
|
157 | process.env.NODE_ENV = settings.environment;
|
158 | }
|
159 |
|
160 | const filters = {
|
161 | ids: settings.ids,
|
162 | grep: settings.grep ? new RegExp(settings.grep) : null
|
163 | };
|
164 |
|
165 | const count = internals.count(experiments, { filters });
|
166 | reporter.start({ count });
|
167 |
|
168 | const startTime = Date.now();
|
169 | const state = {
|
170 | report: {
|
171 | tests: [],
|
172 | failures: 0,
|
173 | errors: []
|
174 | },
|
175 | reporter,
|
176 | filters,
|
177 | options: settings,
|
178 | only: onlyNodes
|
179 | };
|
180 |
|
181 | await internals.executeExperiments(experiments, state, settings.dry);
|
182 | const notebook = {
|
183 | ms: Date.now() - startTime,
|
184 | tests: state.report.tests,
|
185 | failures: state.report.failures,
|
186 | errors: state.report.errors
|
187 | };
|
188 |
|
189 | return Promise.resolve(notebook);
|
190 | };
|
191 |
|
192 |
|
193 | internals.skipAllButOnly = (scripts, onlyNode) => {
|
194 |
|
195 | let currentExperiment = onlyNode.experiment;
|
196 | if (onlyNode.test) {
|
197 | currentExperiment.tests
|
198 | .filter(internals.not(onlyNode.test))
|
199 | .filter(internals.notOnly)
|
200 | .forEach(internals.enableSkip);
|
201 |
|
202 | currentExperiment.experiments
|
203 | .filter(internals.notOnly)
|
204 | .forEach(internals.enableSkip);
|
205 | }
|
206 |
|
207 | while (currentExperiment.parent) {
|
208 | currentExperiment.parent.tests
|
209 | .filter(internals.notOnly)
|
210 | .forEach(internals.enableSkip);
|
211 |
|
212 | currentExperiment.parent.experiments
|
213 | .filter(internals.not(currentExperiment))
|
214 | .filter(internals.notOnly)
|
215 | .filter((experiment) => {
|
216 |
|
217 | return experiment.tests.every(internals.notOnly);
|
218 | })
|
219 | .forEach(internals.enableSkip);
|
220 |
|
221 | currentExperiment = currentExperiment.parent;
|
222 | }
|
223 |
|
224 | scripts.forEach((script) => {
|
225 |
|
226 | if (!script._onlyNodes.length) {
|
227 | internals.enableSkip(script._root);
|
228 | }
|
229 | });
|
230 | };
|
231 |
|
232 | internals.not = (excludedElement) => {
|
233 |
|
234 | return (element) => element !== excludedElement;
|
235 | };
|
236 |
|
237 | internals.notOnly = (element) => {
|
238 |
|
239 | return !element.options.only;
|
240 | };
|
241 |
|
242 | internals.enableSkip = (element) => {
|
243 |
|
244 | element.options.skip = true;
|
245 | };
|
246 |
|
247 | internals.shuffle = function (scripts, seed) {
|
248 |
|
249 | const random = Seedrandom(seed);
|
250 |
|
251 | const last = scripts.length - 1;
|
252 | for (let i = 0; i < scripts.length; ++i) {
|
253 | const rand = i + Math.floor(random() * (last - i + 1));
|
254 | const temp = scripts[i];
|
255 | scripts[i] = scripts[rand];
|
256 | scripts[rand] = temp;
|
257 | }
|
258 | };
|
259 |
|
260 |
|
261 | internals.executeExperiments = async function (experiments, state, skip, parentContext) {
|
262 |
|
263 | for (const experiment of experiments) {
|
264 | const skipExperiment = skip || experiment.options.skip || !internals.experimentHasTests(experiment, state) || (state.options.bail && state.report.failures);
|
265 |
|
266 | state.currentContext = parentContext ? Hoek.clone(parentContext) : {};
|
267 |
|
268 |
|
269 |
|
270 | if (!skipExperiment) {
|
271 | try {
|
272 | await internals.executeDeps(experiment.befores, state);
|
273 | }
|
274 | catch (ex) {
|
275 | internals.fail([experiment], state, skip, '\'before\' action failed');
|
276 | state.report.errors.push(ex);
|
277 |
|
278 |
|
279 | continue;
|
280 | }
|
281 | }
|
282 |
|
283 |
|
284 |
|
285 | await internals.executeTests(experiment, state, skipExperiment);
|
286 |
|
287 |
|
288 |
|
289 | await internals.executeExperiments(experiment.experiments, state, skipExperiment, state.currentContext);
|
290 |
|
291 |
|
292 |
|
293 | if (!skipExperiment) {
|
294 | try {
|
295 | await internals.executeDeps(experiment.afters, state);
|
296 | }
|
297 | catch (ex) {
|
298 | internals.fail([experiment], state, skip, '\'after\' action failed');
|
299 | state.report.errors.push(ex);
|
300 | }
|
301 | }
|
302 | }
|
303 | };
|
304 |
|
305 |
|
306 | internals.executeDeps = async function (deps, state) {
|
307 |
|
308 | if (!deps || !deps.length) {
|
309 | return Promise.resolve();
|
310 | }
|
311 |
|
312 | for (const dep of deps) {
|
313 | dep.options.timeout = Number.isSafeInteger(dep.options.timeout) ? dep.options.timeout : state.options['context-timeout'];
|
314 | await internals.protect(dep, state);
|
315 | }
|
316 | };
|
317 |
|
318 |
|
319 | internals.executeTests = async function (experiment, state, skip) {
|
320 |
|
321 | if (!experiment.tests.length) {
|
322 | return Promise.resolve();
|
323 | }
|
324 |
|
325 |
|
326 |
|
327 | const befores = skip ? [] : internals.collectDeps(experiment, 'beforeEaches');
|
328 | const afters = skip ? [] : internals.collectDeps(experiment, 'afterEaches');
|
329 |
|
330 |
|
331 |
|
332 | const execute = async function (test) {
|
333 |
|
334 | const isNotFiltered = state.filters.ids.length && !state.filters.ids.includes(test.id);
|
335 | const isNotGrepped = state.filters.grep && !state.filters.grep.test(test.title);
|
336 |
|
337 | if (isNotFiltered || isNotGrepped) {
|
338 | return new Promise((resolve) => {
|
339 |
|
340 | setImmediate(resolve);
|
341 | });
|
342 | }
|
343 |
|
344 | const isSkipped = skip || test.options.skip || (state.options.bail && state.report.failures);
|
345 |
|
346 | if (!test.fn ||
|
347 | isSkipped) {
|
348 |
|
349 | test[test.fn ? 'skipped' : 'todo'] = true;
|
350 | test.duration = 0;
|
351 | state.report.tests.push(test);
|
352 | state.reporter.test(test);
|
353 | return new Promise((resolve) => {
|
354 |
|
355 | setImmediate(resolve);
|
356 | });
|
357 | }
|
358 |
|
359 |
|
360 |
|
361 | try {
|
362 | await internals.executeDeps(befores, state);
|
363 | }
|
364 | catch (ex) {
|
365 | internals.failTest(test, state, skip, ex);
|
366 | state.report.errors.push(ex);
|
367 | return Promise.resolve();
|
368 | }
|
369 |
|
370 |
|
371 |
|
372 | const start = Date.now();
|
373 | try {
|
374 | test.context = Hoek.clone(state.currentContext);
|
375 | await internals.protect(test, state);
|
376 | }
|
377 | catch (ex) {
|
378 | state.report.failures++;
|
379 | test.err = ex;
|
380 | test.timeout = ex.timeout;
|
381 | }
|
382 |
|
383 | test.duration = Date.now() - start;
|
384 |
|
385 | state.report.tests.push(test);
|
386 | state.reporter.test(test);
|
387 |
|
388 |
|
389 |
|
390 | try {
|
391 | await internals.executeDeps(afters, state);
|
392 | }
|
393 | catch (ex) {
|
394 | state.report.failures++;
|
395 | state.report.errors.push(ex);
|
396 | }
|
397 |
|
398 | return Promise.resolve();
|
399 | };
|
400 |
|
401 | for (const test of experiment.tests) {
|
402 | await execute(test);
|
403 | }
|
404 |
|
405 | return Promise.resolve();
|
406 | };
|
407 |
|
408 |
|
409 | internals.experimentHasTests = function (experiment, state) {
|
410 |
|
411 | if (experiment.experiments.length) {
|
412 | const experimentsHasTests = experiment.experiments.some((childExperiment) => {
|
413 |
|
414 | return internals.experimentHasTests(childExperiment, state);
|
415 | });
|
416 |
|
417 | if (experimentsHasTests) {
|
418 | return true;
|
419 | }
|
420 | }
|
421 |
|
422 | const hasTests = experiment.tests.some((test) => {
|
423 |
|
424 | if ((state.filters.ids.length && state.filters.ids.indexOf(test.id) === -1) ||
|
425 | (state.filters.grep && !state.filters.grep.test(test.title))) {
|
426 |
|
427 | return false;
|
428 | }
|
429 |
|
430 | if (!test.options.skip && test.fn) {
|
431 | return true;
|
432 | }
|
433 | });
|
434 |
|
435 | return hasTests;
|
436 | };
|
437 |
|
438 |
|
439 | internals.collectDeps = function (experiment, key) {
|
440 |
|
441 | const set = [];
|
442 |
|
443 | // if we are looking at afterEaches, we want to run our parent's blocks before ours (unshift onto front)
|
444 | const arrayAddFn = key === 'afterEaches' ? Array.prototype.unshift : Array.prototype.push;
|
445 |
|
446 | if (experiment.parent) {
|
447 | arrayAddFn.apply(set, internals.collectDeps(experiment.parent, key));
|
448 | }
|
449 |
|
450 | arrayAddFn.apply(set, experiment[key] || []);
|
451 | return set;
|
452 | };
|
453 |
|
454 |
|
455 | internals.createItemTimeout = (item, ms, finish) => {
|
456 |
|
457 | return setTimeout(() => {
|
458 |
|
459 | const error = new Error(`Timed out (${ms}ms) - ${item.title}`);
|
460 | error.timeout = true;
|
461 | finish(error, 'timeout');
|
462 | }, ms);
|
463 | };
|
464 |
|
465 |
|
466 | internals.protect = function (item, state) {
|
467 |
|
468 | let isFirst = true;
|
469 | let timeoutId;
|
470 | let countBefore = -1;
|
471 | let failedWithUnhandledRejection = false;
|
472 | let failedWithUncaughtException = false;
|
473 |
|
474 | if (state.options.assert && state.options.assert.count) {
|
475 | countBefore = state.options.assert.count();
|
476 | }
|
477 |
|
478 | return new Promise((resolve, reject) => {
|
479 |
|
480 | const flags = { notes: [], context: item.context || state.currentContext };
|
481 | flags.note = (note) => {
|
482 |
|
483 | flags.notes.push(note);
|
484 | };
|
485 |
|
486 | const willcall = new WillCall();
|
487 | flags.mustCall = willcall.expect.bind(willcall);
|
488 |
|
489 | const finish = async function (err, cause) {
|
490 |
|
491 | clearTimeout(timeoutId);
|
492 | timeoutId = null;
|
493 |
|
494 | item.notes = (item.notes || []).concat(flags.notes);
|
495 |
|
496 | process.removeListener('unhandledRejection', promiseRejectionHandler);
|
497 | process.removeListener('uncaughtException', processUncaughtExceptionHandler);
|
498 |
|
499 | if (cause && (err instanceof Error === false)) {
|
500 | const data = err;
|
501 | err = new Error(`Non Error object received or caught (${cause})`);
|
502 | err.data = data;
|
503 | }
|
504 |
|
505 |
|
506 |
|
507 | if (failedWithUnhandledRejection || failedWithUncaughtException) {
|
508 | return reject(err);
|
509 | }
|
510 |
|
511 |
|
512 | if (state.options.assert && state.options.assert.count) {
|
513 | item.assertions = state.options.assert.count() - countBefore;
|
514 | }
|
515 |
|
516 | if (flags.onCleanup) {
|
517 |
|
518 |
|
519 | const onCleanupError = (err) => {
|
520 |
|
521 | return reject(err);
|
522 | };
|
523 |
|
524 |
|
525 | process.once('uncaughtException', onCleanupError);
|
526 |
|
527 | try {
|
528 | await flags.onCleanup();
|
529 | }
|
530 | catch (ex) {
|
531 | return reject(ex);
|
532 | }
|
533 |
|
534 | process.removeListener('uncaughtException', onCleanupError);
|
535 | }
|
536 |
|
537 | if (item.options.plan !== undefined) {
|
538 | if (item.options.plan !== item.assertions) {
|
539 | const planMessage = (item.assertions === undefined)
|
540 | ? `Expected ${item.options.plan} assertions, but no assertion library found`
|
541 | : `Expected ${item.options.plan} assertions, but found ${item.assertions}`;
|
542 | if (err && !/^Expected (at least )?\d+ assertions/.test(err.message)) {
|
543 | err.message = planMessage + ': ' + err.message;
|
544 | }
|
545 | else {
|
546 | err = new Error(planMessage);
|
547 | }
|
548 | }
|
549 | }
|
550 | else if (item.path &&
|
551 | state.options['default-plan-threshold'] &&
|
552 | (item.assertions === undefined ||
|
553 | item.assertions < state.options['default-plan-threshold'])) {
|
554 | const planMessage = (item.assertions === undefined)
|
555 | ? `Expected at least ${state.options['default-plan-threshold']} assertions, but no assertion library found`
|
556 | : `Expected at least ${state.options['default-plan-threshold']} assertions, but found ${item.assertions}`;
|
557 | if (err && !/^Expected (at least )?\d+ assertions/.test(err.message)) {
|
558 | err.message = planMessage + ': ' + err.message;
|
559 | }
|
560 | else {
|
561 | err = new Error(planMessage);
|
562 | }
|
563 | }
|
564 |
|
565 | if (!isFirst) {
|
566 | const message = `Thrown error received in test "${item.title}" (${cause})`;
|
567 | err = new Error(message);
|
568 | }
|
569 |
|
570 | isFirst = false;
|
571 |
|
572 | const callResults = willcall.check();
|
573 | if (callResults.length) {
|
574 | const callResult = callResults[0];
|
575 | err = new Error(`Expected ${callResult.name} to be executed ${callResult.expected}` +
|
576 | `time(s) but was executed ${callResult.actual} time(s)`);
|
577 | err.stack = callResult.stack;
|
578 | }
|
579 |
|
580 | item.context = null;
|
581 |
|
582 | return err ? reject(err) : resolve();
|
583 | };
|
584 |
|
585 | const ms = item.options.timeout !== undefined ? item.options.timeout : state.options.timeout;
|
586 |
|
587 |
|
588 |
|
589 | const promiseRejectionHandler = function (err) {
|
590 |
|
591 | if (flags.onUnhandledRejection) {
|
592 | flags.onUnhandledRejection(err);
|
593 | return;
|
594 | }
|
595 |
|
596 | failedWithUnhandledRejection = true;
|
597 | finish(err, 'unhandledRejection');
|
598 | };
|
599 |
|
600 | process.on('unhandledRejection', promiseRejectionHandler);
|
601 |
|
602 | const processUncaughtExceptionHandler = function (err) {
|
603 |
|
604 | if (flags.onUncaughtException) {
|
605 | try {
|
606 | flags.onUncaughtException(err);
|
607 | return;
|
608 | }
|
609 | catch (errInErrorhandling) {
|
610 | err = errInErrorhandling;
|
611 | }
|
612 | }
|
613 |
|
614 | failedWithUncaughtException = true;
|
615 | finish(err, 'uncaughtException');
|
616 | };
|
617 |
|
618 |
|
619 | process.on('uncaughtException', processUncaughtExceptionHandler);
|
620 |
|
621 | setImmediate(async () => {
|
622 |
|
623 | if (ms) {
|
624 | timeoutId = internals.createItemTimeout(item, ms, finish);
|
625 | }
|
626 |
|
627 | try {
|
628 | await item.fn.call(null, flags);
|
629 | finish();
|
630 | }
|
631 | catch (ex) {
|
632 | return finish(ex, 'unit test');
|
633 | }
|
634 | });
|
635 | });
|
636 | };
|
637 |
|
638 |
|
639 | internals.count = function (experiments, state) {
|
640 |
|
641 | state.count = state.count || 0;
|
642 | state.seq = state.seq || 0;
|
643 |
|
644 | for (let i = 0; i < experiments.length; ++i) {
|
645 | const experiment = experiments[i];
|
646 |
|
647 | for (let j = 0; j < experiment.tests.length; ++j) {
|
648 | const test = experiment.tests[j];
|
649 | test.id = ++state.seq;
|
650 | state.count += (state.filters.ids.length && state.filters.ids.indexOf(test.id) === -1) || (state.filters.grep && !state.filters.grep.test(test.title)) ? 0 : 1;
|
651 | }
|
652 |
|
653 | internals.count(experiment.experiments, state);
|
654 | }
|
655 |
|
656 | return state.count;
|
657 | };
|
658 |
|
659 |
|
660 | internals.fail = function (experiments, state, skip, err) {
|
661 |
|
662 | for (let i = 0; i < experiments.length; ++i) {
|
663 | const experiment = experiments[i];
|
664 |
|
665 | for (let j = 0; j < experiment.tests.length; ++j) {
|
666 | internals.failTest(experiment.tests[j], state, skip, err);
|
667 | }
|
668 |
|
669 | internals.fail(experiment.experiments, state, skip || experiment.options.skip, err);
|
670 | }
|
671 | };
|
672 |
|
673 |
|
674 | internals.failTest = function (test, state, skip, err) {
|
675 |
|
676 | if (!test.fn ||
|
677 | skip ||
|
678 | test.options.skip) {
|
679 |
|
680 | test[test.fn ? 'skipped' : 'todo'] = true;
|
681 | }
|
682 | else {
|
683 | state.report.failures++;
|
684 | test.err = err;
|
685 | }
|
686 |
|
687 | test.duration = 0;
|
688 |
|
689 | state.report.tests.push(test);
|
690 | state.reporter.test(test);
|
691 | };
|