1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 |
|
18 | var util = require('util');
|
19 | var fs = require('fs');
|
20 | var path = require('path');
|
21 | var net = require('net');
|
22 | var spawn = require('child_process').spawn;
|
23 | var exec = require('child_process').exec;
|
24 | var execSync = require('child_process').execSync;
|
25 |
|
26 | var sprintf = require('sprintf').sprintf;
|
27 | var async = require('async');
|
28 | var logmagic = require('logmagic');
|
29 | var underscore = require('underscore');
|
30 |
|
31 | var constants = require('./constants');
|
32 | var parser = require('./parser');
|
33 | var common = require('./common');
|
34 | var testUtil = require('./util');
|
35 | var coverage = require('./coverage');
|
36 | var getReporter = require('./reporters/index').getReporter;
|
37 | var generateMakefile = require('./gen_makefile').generateMakefile;
|
38 | var _debugger = require('./debugger');
|
39 | var DebuggerInterface = _debugger.Interface;
|
40 | var ProcessRunner = require('./process_runner/runner').ProcessRunner;
|
41 |
|
42 | exports.exitCode = 0;
|
43 | exports.processRunner = null;
|
44 |
|
45 | function TestRunner(options) {
|
46 | var testReporter, coverageReporter, scopeLeaksReporter,
|
47 | testReporterOptions, coverageReporterOptions, scopeLeaksReporterOptions,
|
48 | defaultSocketPath;
|
49 |
|
50 | defaultSocketPath = sprintf('%s-%s', constants.DEFAULT_SOCKET_PATH,
|
51 | Math.random() * 10000);
|
52 | this._tests = options['tests'];
|
53 | this._independent_tests = options['independent-tests'];
|
54 | this._max_suites = options['max-suites'];
|
55 | this._dependencies = options['dependencies'] || null;
|
56 | this._onlyEssential = options['only-essential'] || false;
|
57 | this._socketPath = options['socket-path'] || defaultSocketPath;
|
58 | this._coverage = options['coverage'];
|
59 | this._scopeLeaks = options['scope-leaks'];
|
60 | this._verbosity = options['verbosity'] || constants.DEFAULT_VERBOSITY;
|
61 | this._failfast = options['failfast'] || false;
|
62 | this._realTime = options['real-time'] || false;
|
63 | this._debug = options['debug'];
|
64 |
|
65 | testReporter = options['test-reporter'] || constants.DEFAULT_TEST_REPORTER;
|
66 | testReporterOptions = {
|
67 | 'print_stdout': options['print-stdout'],
|
68 | 'print_stderr': options['print-stderr'],
|
69 | 'styles': (options['no-styles'] ? false : true),
|
70 | 'report_timing': options['report-timing']
|
71 | };
|
72 |
|
73 | this._testReporter = getReporter('test', testReporter, this._tests,
|
74 | testReporterOptions);
|
75 |
|
76 | if (this._coverage) {
|
77 | coverageReporter = options['coverage-reporter'] || constants.DEFAULT_COVERAGE_REPORTER;
|
78 | coverageReporterOptions = {
|
79 | 'directory': options['coverage-dir'],
|
80 | 'file': options['coverage-file']
|
81 | };
|
82 |
|
83 | this._coverageReporter = getReporter('coverage',
|
84 | coverageReporter,
|
85 | this._tests,
|
86 | coverageReporterOptions);
|
87 | }
|
88 | else {
|
89 | this._coverageReporter = null;
|
90 | }
|
91 |
|
92 | if (this._scopeLeaks) {
|
93 | scopeLeaksReporter = options['scope-leaks-reporter'] || constants.DEFAULT_SCOPE_LEAKS_REPORTER;
|
94 | scopeLeaksReporterOptions = {
|
95 | 'sequential': (options['concurrency'] === 1 || options['sequential'])
|
96 | };
|
97 | this._scopeLeaksReporter = getReporter('scope-leaks', scopeLeaksReporter,
|
98 | this._tests,
|
99 | scopeLeaksReporterOptions);
|
100 | }
|
101 | else {
|
102 | this._scopeLeaksReporter = null;
|
103 | }
|
104 |
|
105 | this._server = null;
|
106 | this._processRunner = null;
|
107 | this._testFilesData = {};
|
108 |
|
109 | this._completed = false;
|
110 | this._forceStopped = false;
|
111 | this._completedTests = [];
|
112 | }
|
113 |
|
114 | TestRunner.prototype.runTests = function(testInitFile, chdir,
|
115 | customAssertModule,
|
116 | timeout,
|
117 | concurrency, failFast) {
|
118 | var self = this, ops = [];
|
119 | timeout = timeout || constants.DEFAULT_TEST_TIMEOUT;
|
120 |
|
121 | function handleChildTimeout(child, filePath) {
|
122 | var resultObj, testFile;
|
123 |
|
124 | if (!child.killed) {
|
125 | testFile = self._testFilesData[filePath];
|
126 |
|
127 | resultObj = {
|
128 | 'tests': [],
|
129 | 'timeout': true
|
130 | };
|
131 |
|
132 | child.kill('SIGKILL');
|
133 | self._handleTestResult(filePath, resultObj);
|
134 | }
|
135 |
|
136 | if (self._failfast) {
|
137 | self.forceStop();
|
138 | }
|
139 | }
|
140 |
|
141 | function runSuite(filePath, callback) {
|
142 | var result, pattern, cwd, child, timeoutId, testFileData;
|
143 |
|
144 | if (self._completed || self._forceStopped) {
|
145 | callback(new Error('Runner has been stopped'));
|
146 | return;
|
147 | }
|
148 |
|
149 | cwd = process.cwd();
|
150 | result = common.getTestFilePathAndPattern(filePath);
|
151 | filePath = result[0];
|
152 | pattern = result[1];
|
153 | filePath = (filePath.charAt(0) !== '/') ? path.join(cwd, filePath) : filePath;
|
154 | child = self._spawnTestProcess(filePath, testInitFile, chdir,
|
155 | customAssertModule, timeout,
|
156 | concurrency, pattern);
|
157 |
|
158 | if (!self._debug) {
|
159 | timeoutId = setTimeout(function() {
|
160 | handleChildTimeout(child, filePath, callback);
|
161 | }, timeout);
|
162 | }
|
163 |
|
164 | self._testFilesData[filePath] = {
|
165 | 'child': child,
|
166 | 'callback': callback,
|
167 | 'timeout_id': timeoutId,
|
168 | 'stdout': '',
|
169 | 'stderr': ''
|
170 | };
|
171 |
|
172 | testFileData = self._testFilesData[filePath];
|
173 |
|
174 | child.stdout.on('data', function(chunk) {
|
175 | if (self._realTime) {
|
176 | process.stdout.write(chunk.toString());
|
177 | }
|
178 | else {
|
179 | testFileData['stdout'] += chunk;
|
180 | }
|
181 | });
|
182 |
|
183 | child.stderr.on('data', function(chunk) {
|
184 | if (self._realTime) {
|
185 | process.stderr.write(chunk.toString());
|
186 | }
|
187 | else {
|
188 | testFileData['stderr'] += chunk;
|
189 | }
|
190 | });
|
191 |
|
192 | child.on('exit', function() {
|
193 | clearTimeout(timeoutId);
|
194 | self._handleTestFileEnd(filePath);
|
195 | delete self._testFilesData[filePath];
|
196 | });
|
197 | }
|
198 |
|
199 | function onBound() {
|
200 | self._testReporter.handleTestsStart();
|
201 | async.series([
|
202 | async.forEachLimit.bind(null, self._independent_tests, self._max_suites, runSuite),
|
203 | async.forEachSeries.bind(null, self._tests, runSuite)
|
204 | ], self._handleTestsCompleted.bind(self));
|
205 | }
|
206 |
|
207 | function startServer(callback) {
|
208 | self._startServer(self._handleConnection.bind(self),
|
209 | onBound);
|
210 | }
|
211 |
|
212 | function createRunner(callback) {
|
213 | self._processRunner = new ProcessRunner(self._dependencies);
|
214 | exports.processRunner = self._processRunner;
|
215 | callback(null, null);
|
216 | }
|
217 |
|
218 | function findDependenciesToRun(_, callback) {
|
219 | var cwd = process.cwd();
|
220 | var testPaths = self._tests.map(function(testPath) {
|
221 | testPath = common.getTestFilePathAndPattern(testPath)[0];
|
222 | testPath = (testPath.charAt(0) !== '/') ? path.join(cwd, testPath) : testPath;
|
223 | return testPath;
|
224 | });
|
225 |
|
226 | self._processRunner.findDependencies(testPaths, callback);
|
227 | }
|
228 |
|
229 | function startDependencies(dependencies, callback) {
|
230 | self._processRunner.start(dependencies, callback);
|
231 | }
|
232 |
|
233 | if (self._dependencies) {
|
234 | ops.push(createRunner);
|
235 |
|
236 | if (self._onlyEssential) {
|
237 | ops.push(findDependenciesToRun);
|
238 | }
|
239 |
|
240 | ops.push(startDependencies);
|
241 | }
|
242 |
|
243 | ops.push(startServer);
|
244 |
|
245 | async.waterfall(ops, function(err) {
|
246 | if (err) {
|
247 | throw err;
|
248 | }
|
249 | });
|
250 | };
|
251 |
|
252 | TestRunner.prototype._spawnTestProcess = function(filePath,
|
253 | testInitFile,
|
254 | chdir,
|
255 | customAssertModule,
|
256 | timeout,
|
257 | concurrency,
|
258 | pattern) {
|
259 | var cwd = process.cwd();
|
260 | var libCovDir = (this._coverage) ? path.join(cwd, 'lib-cov') : null;
|
261 | var runFilePath = path.join(__dirname, 'run_test_file.js');
|
262 | var port = parseInt((Math.random() * (65000 - 2000) + 2000), 10);
|
263 | var args = [];
|
264 |
|
265 | if (this._debug) {
|
266 | args.push('--debug-brk=' + port);
|
267 | }
|
268 |
|
269 | args = args.concat([runFilePath, filePath, this._socketPath, cwd, libCovDir,
|
270 | this._scopeLeaks, chdir, customAssertModule, testInitFile,
|
271 | timeout, concurrency, pattern]);
|
272 | this._testReporter.handleTestFileStart(filePath);
|
273 | var child = spawn(process.execPath, args);
|
274 |
|
275 | if (this._debug) {
|
276 | var debuggerInterface = new DebuggerInterface();
|
277 | debuggerInterface.connect(port, 400, function(err, client) {
|
278 |
|
279 | client.reqContinue();
|
280 | });
|
281 | }
|
282 |
|
283 | return child;
|
284 | };
|
285 |
|
286 | TestRunner.prototype._addCompletedTest = function(filePath) {
|
287 | this._completedTests.push(filePath);
|
288 | };
|
289 |
|
290 | TestRunner.prototype._handleTestsCompleted = function() {
|
291 | var statusCode;
|
292 |
|
293 | if (!this._completed || this._forceStopped) {
|
294 | this._completed = true;
|
295 | this._stopServer();
|
296 |
|
297 | statusCode = this._testReporter.handleTestsComplete();
|
298 |
|
299 | if (this._scopeLeaks) {
|
300 | this._scopeLeaksReporter.handleTestsComplete();
|
301 | }
|
302 |
|
303 | if (this._coverage) {
|
304 | this._coverageReporter.handleTestsComplete();
|
305 | }
|
306 |
|
307 | exports.exitCode = statusCode;
|
308 |
|
309 | if (this._processRunner) {
|
310 | this._processRunner.stop();
|
311 | }
|
312 |
|
313 | if (this._debug) {
|
314 | process.exit();
|
315 | }
|
316 | }
|
317 | };
|
318 |
|
319 | TestRunner.prototype._handleTestResult = function(filePath, resultObj) {
|
320 | this._testReporter.handleTestEnd(filePath, resultObj);
|
321 |
|
322 | if (this._scopeLeaks) {
|
323 | this._scopeLeaksReporter.handleTestEnd(filePath, resultObj);
|
324 | }
|
325 |
|
326 | if (this._failfast && resultObj['status'] === 'failure') {
|
327 | this.forceStop();
|
328 | }
|
329 | };
|
330 |
|
331 | TestRunner.prototype._handleTestCoverageResult = function(filePath, coverageData) {
|
332 | var coverageObj = JSON.parse(coverageData);
|
333 | this._coverageReporter.handleTestFileComplete(filePath, coverageObj);
|
334 | };
|
335 |
|
336 | TestRunner.prototype._handleTestFileEnd = function(filePath) {
|
337 | var testData, coverageData, resultObj, coverageObj, split, testFile;
|
338 | var stdout = '';
|
339 | var stderr = '';
|
340 |
|
341 | testFile = this._testFilesData[filePath];
|
342 | if (testFile) {
|
343 | stdout = testFile['stdout'];
|
344 | stderr = testFile['stderr'];
|
345 | }
|
346 |
|
347 | this._testReporter.handleTestFileComplete(filePath, stdout, stderr);
|
348 | this._addCompletedTest(filePath);
|
349 |
|
350 | if (testFile) {
|
351 | testFile['callback']();
|
352 | }
|
353 | };
|
354 |
|
355 | TestRunner.prototype._handleConnection = function(connection) {
|
356 | var self = this;
|
357 | var data = '';
|
358 | var lineProcessor = new testUtil.LineProcessor();
|
359 | var dataString, endMarkIndex, testFile, testFileCallback, testFileTimeoutId;
|
360 |
|
361 | function onLine(line) {
|
362 | var result, filePath, end, resultObj;
|
363 | result = testUtil.parseResultLine(line);
|
364 | end = result[0];
|
365 | filePath = result[1];
|
366 | resultObj = result[2];
|
367 |
|
368 | if (end) {
|
369 | return;
|
370 | }
|
371 |
|
372 | if (resultObj.hasOwnProperty('coverage')) {
|
373 | self._handleTestCoverageResult(filePath, resultObj['coverage']);
|
374 | }
|
375 | else {
|
376 | self._handleTestResult(filePath, resultObj);
|
377 | }
|
378 | }
|
379 |
|
380 | lineProcessor.on('line', onLine);
|
381 |
|
382 | function onData(chunk) {
|
383 | lineProcessor.appendData(chunk);
|
384 | data += chunk;
|
385 | }
|
386 |
|
387 | connection.on('data', onData);
|
388 | };
|
389 |
|
390 | TestRunner.prototype._startServer = function(connectionHandler,
|
391 | onBound) {
|
392 | this._server = net.createServer(connectionHandler);
|
393 | this._server.listen(this._socketPath, onBound);
|
394 | };
|
395 |
|
396 | TestRunner.prototype.forceStop = function() {
|
397 | var testFile, testFileData, child, timeoutId;
|
398 |
|
399 | this._forceStopped = true;
|
400 | for (testFile in this._testFilesData) {
|
401 | if (this._testFilesData.hasOwnProperty(testFile)) {
|
402 | testFileData = this._testFilesData[testFile];
|
403 | child = testFileData['child'];
|
404 | timeoutId = testFileData['timeout_id'];
|
405 | clearTimeout(timeoutId);
|
406 | child.kill('SIGKILL');
|
407 | }
|
408 | }
|
409 | };
|
410 |
|
411 | TestRunner.prototype._stopServer = function() {
|
412 | if (this._server) {
|
413 | this._server.close();
|
414 | this._server = null;
|
415 | }
|
416 | };
|
417 |
|
418 | function run(cwd, argv) {
|
419 | var customAssertModule, exportedFunctions;
|
420 | var runner, testReporter, coverageArgs;
|
421 | var socketPath, concurrency, scopeLeaks, runnerArgs;
|
422 | var intersection;
|
423 |
|
424 | if ((argv === undefined) && (cwd instanceof Array)) {
|
425 | argv = cwd;
|
426 | }
|
427 |
|
428 | var p = parser.getParser(constants.WHISKEY_OPTIONS);
|
429 | p.banner = 'Usage: whiskey [options] --tests "files"';
|
430 | var options = parser.parseArgv(p, argv);
|
431 |
|
432 | if (options['coverage-files']) {
|
433 | var coverageReporter = options['coverage-reporter'] || constants.DEFAULT_COVERAGE_REPORTER;
|
434 | var coverageReporterOptions = {
|
435 | 'directory': options['coverage-dir'],
|
436 | 'file': options['coverage-file']
|
437 | };
|
438 |
|
439 | this._coverageReporter = getReporter('coverage',
|
440 | coverageReporter,
|
441 | [],
|
442 | coverageReporterOptions);
|
443 | var coverageObj = coverage.aggregateCoverage(options['coverage-files'].split(','));
|
444 | this._coverageReporter.handleTestsComplete(coverageObj);
|
445 | }
|
446 | else if ((options.tests && options.tests.length > 0) ||
|
447 | (options['independent-tests'] && options['independent-tests'].length > 0)) {
|
448 | options.tests = options.tests ? options.tests.split(' ') : [];
|
449 | options['independent-tests'] = options['independent-tests'] ? options['independent-tests'].split(' ') : [];
|
450 |
|
451 | var ms = parseInt(options['max-suites'], 10);
|
452 | if (ms) {
|
453 | options['max-suites'] = ms;
|
454 | } else {
|
455 | options['max-suites'] = 5;
|
456 | }
|
457 |
|
458 | intersection = underscore.intersection(options.tests, options['independent-tests']);
|
459 | if(intersection.length > 0) {
|
460 | util.puts(sprintf('The following tests cannot appear in both --tests and --independen-tests: %s', intersection));
|
461 | process.exit(1);
|
462 | }
|
463 |
|
464 | if (options['debug'] && options.tests.length > 1) {
|
465 | throw new Error('--debug option can currently only be used with a single test file');
|
466 | }
|
467 | else if (options['debug'] && options['independent-tests'].length > 1) {
|
468 | throw new Error('--debug cannot be used with --independent-tests.');
|
469 | }
|
470 | else if (options['gen-makefile'] && options['makefile-path']) {
|
471 | generateMakefile(options.tests, options['makefile-path'], function(err) {
|
472 | if (err) {
|
473 | throw err;
|
474 | }
|
475 |
|
476 | util.puts(sprintf('Makefile has been saved in %s.', options['makefile-path']));
|
477 | });
|
478 |
|
479 | return;
|
480 | }
|
481 |
|
482 | customAssertModule = options['custom-assert-module'];
|
483 | if (customAssertModule) {
|
484 | customAssertModule = (customAssertModule.charAt(0) !== '/') ? path.join(cwd, customAssertModule) : customAssertModule;
|
485 |
|
486 | if (path.existsSync(customAssertModule)) {
|
487 | customAssertModule = customAssertModule.replace(/$\.js/, '');
|
488 | }
|
489 | else {
|
490 | customAssertModule = null;
|
491 | }
|
492 | }
|
493 |
|
494 | concurrency = options['sequential'] ? 1 : options['concurrency'];
|
495 | concurrency = concurrency || constants.DEFAULT_CONCURRENCY;
|
496 | scopeLeaks = options['scope-leaks'];
|
497 |
|
498 | options['print-stdout'] = (options['quiet']) ? false : true;
|
499 | options['print-stderr'] = (options['quiet']) ? false : true;
|
500 |
|
501 | if (options['quiet']) {
|
502 | logmagic.registerSink('null', function nullLogger() {});
|
503 | logmagic.route('__root__', logmagic.INFO, 'null');
|
504 | }
|
505 |
|
506 | runner = new TestRunner(options);
|
507 | runnerArgs = [options['test-init-file'], options['chdir'],
|
508 | customAssertModule, options['timeout'],
|
509 | concurrency, options['fail-fast']];
|
510 |
|
511 |
|
512 | if (options['coverage']) {
|
513 | var nodePath = process.env['NODE_PATH'];
|
514 |
|
515 | if (!nodePath || nodePath.indexOf('lib-cov') === -1) {
|
516 | throw new Error('lib-cov is not in NODE_PATH. NODE_PATH environment variable' +
|
517 | ' must contain lib-cov path for the coverage to work.');
|
518 | }
|
519 |
|
520 | coverageArgs = ['jscoverage'];
|
521 |
|
522 | if (options['coverage-encoding']) {
|
523 | coverageArgs.push(sprintf('--encoding=%s', options['coverage-encoding']));
|
524 | }
|
525 |
|
526 | if (options['coverage-exclude']) {
|
527 | coverageArgs.push(sprintf('--exclude=%s', options['coverage-exclude']));
|
528 | }
|
529 |
|
530 | if (options['coverage-no-instrument']) {
|
531 | coverageArgs.push(sprintf('--no-instrument=%s', options['coverage-no-instrument']));
|
532 | }
|
533 |
|
534 | coverageArgs.push(sprintf('lib %s', constants.COVERAGE_PATH));
|
535 | coverageArgs = coverageArgs.join(' ');
|
536 |
|
537 | if (!path.existsSync(path.join(process.cwd(), constants.COVERAGE_PATH)) || !options['coverage-no-regen']) {
|
538 | exec(sprintf('rm -fr %s ; %s', constants.COVERAGE_PATH, coverageArgs), function(err) {
|
539 | if (err) {
|
540 | if (err.message.match(/jscoverage: not found/i)) {
|
541 | err = new Error('jscoverage binary not found. To use test coverage ' +
|
542 | ' you need to install node-jscoverag binary - ' +
|
543 | 'https://github.com/visionmedia/node-jscoverage');
|
544 | }
|
545 |
|
546 | throw err;
|
547 | }
|
548 |
|
549 | runner.runTests.apply(runner, runnerArgs);
|
550 | });
|
551 | }
|
552 | else {
|
553 | runner.runTests.apply(runner, runnerArgs);
|
554 | }
|
555 | }
|
556 | else {
|
557 | runner.runTests.apply(runner, runnerArgs);
|
558 | }
|
559 |
|
560 | }
|
561 | else if (!p._halted) {
|
562 | console.log(p.banner);
|
563 | }
|
564 |
|
565 | process.on('SIGINT', function onSigint() {
|
566 | runner.forceStop();
|
567 | });
|
568 |
|
569 | process.on('exit', function() {
|
570 | process.reallyExit(exports.exitCode);
|
571 | });
|
572 | }
|
573 |
|
574 | exports.run = run;
|