intern/lib/reporters/Benchmark.js

(function (factory) {
    if (typeof module === "object" && typeof module.exports === "object") {
        var v = factory(require, exports);
        if (v !== undefined) module.exports = v;
    }
    else if (typeof define === "function" && define.amd) {
        define(["require", "exports", "tslib", "fs", "./Reporter"], factory);
    }
})(function (require, exports) {
    "use strict";
    Object.defineProperty(exports, "__esModule", { value: true });
    var tslib_1 = require("tslib");
    var fs_1 = require("fs");
    var Reporter_1 = tslib_1.__importStar(require("./Reporter"));
    var BenchmarkReporter = (function (_super) {
        tslib_1.__extends(BenchmarkReporter, _super);
        function BenchmarkReporter(executor, options) {
            if (options === void 0) { options = {}; }
            var _this = _super.call(this, executor, options) || this;
            _this.mode = options.mode || 'test';
            _this.filename = options.filename || '';
            _this.thresholds = options.thresholds || {};
            if (_this.mode === 'test') {
                try {
                    _this.baseline = JSON.parse(fs_1.readFileSync(_this.filename, { encoding: 'utf8' }));
                }
                catch (error) {
                    _this.console.warn('Unable to load benchmark baseline data from ' + _this.filename);
                    _this.console.warn('Switching to "baseline" mode');
                    _this.mode = 'baseline';
                }
            }
            if (!_this.baseline) {
                _this.baseline = {};
            }
            _this.sessions = {};
            return _this;
        }
        BenchmarkReporter.prototype._getSession = function (testOrSuite) {
            var sessionId = testOrSuite.sessionId || 'local';
            var session = this.sessions[sessionId];
            if (!session) {
                var client = void 0;
                var version = void 0;
                var platform_1;
                if (testOrSuite.sessionId) {
                    var environmentType = testOrSuite.remote.environmentType;
                    client = environmentType.browserName;
                    version = environmentType.version;
                    platform_1 = environmentType.platform;
                }
                else {
                    client = process.title;
                    version = process.version;
                    platform_1 = process.platform;
                }
                session = this.sessions[sessionId] = {
                    suites: {},
                    environment: {
                        client: client,
                        version: version,
                        platform: platform_1,
                        id: client + ':' + version + ':' + platform_1
                    }
                };
            }
            return session;
        };
        BenchmarkReporter.prototype.runEnd = function () {
            if (this.mode === 'baseline') {
                var existingBaseline_1;
                try {
                    existingBaseline_1 = JSON.parse(fs_1.readFileSync(this.filename, { encoding: 'utf8' }));
                }
                catch (error) {
                    existingBaseline_1 = {};
                }
                var baseline_1 = this.baseline;
                Object.keys(baseline_1).forEach(function (environmentId) {
                    existingBaseline_1[environmentId] = baseline_1[environmentId];
                });
                fs_1.writeFileSync(this.filename, JSON.stringify(existingBaseline_1, null, '    '));
            }
        };
        BenchmarkReporter.prototype.suiteEnd = function (suite) {
            var session = this._getSession(suite);
            if (!suite.hasParent) {
                var environment = session.environment;
                this.console.log('Finished benchmarking ' +
                    environment.client +
                    ' ' +
                    environment.version +
                    ' on ' +
                    environment.platform);
            }
            else if (this.mode === 'test') {
                var suiteInfo = session.suites[suite.id];
                var numTests = suiteInfo.numBenchmarks;
                if (numTests > 0) {
                    var numFailedTests = suiteInfo.numFailedBenchmarks;
                    var message = numFailedTests + '/' + numTests + ' benchmarks failed in ' + suite.id;
                    if (numFailedTests > 0) {
                        this.console.warn(message);
                    }
                    else {
                        this.console.log(message);
                    }
                }
            }
        };
        BenchmarkReporter.prototype.suiteStart = function (suite) {
            var session = this._getSession(suite);
            if (!suite.hasParent) {
                var environment = session.environment;
                var environmentName = environment.client +
                    ' ' +
                    environment.version +
                    ' on ' +
                    environment.platform;
                var baselineEnvironments = this.baseline;
                this.console.log((this.mode === 'baseline' ? 'Baselining' : 'Benchmarking') +
                    ' ' +
                    environmentName);
                if (this.mode === 'baseline') {
                    baselineEnvironments[environment.id] = {
                        client: environment.client,
                        version: environment.version,
                        platform: environment.platform,
                        tests: {}
                    };
                }
                else if (!baselineEnvironments[environment.id]) {
                    this.console.warn('No baseline data for ' + environmentName + '!');
                }
            }
            else {
                session.suites[suite.id] = {
                    numBenchmarks: 0,
                    numFailedBenchmarks: 0
                };
            }
        };
        BenchmarkReporter.prototype.testEnd = function (test) {
            var _this = this;
            var benchmarkTest = test;
            if (benchmarkTest.benchmark == null) {
                return;
            }
            if (benchmarkTest.error) {
                var session = this._getSession(benchmarkTest);
                var suiteInfo = session.suites[benchmarkTest.parentId];
                suiteInfo.numBenchmarks++;
                suiteInfo.numFailedBenchmarks++;
                this.console.error('FAIL: ' + benchmarkTest.id);
                this.console.error(this.executor.formatError(benchmarkTest.error, { space: '  ' }));
            }
            else {
                var checkTest = function (baseline, benchmark) {
                    var warn = [];
                    var fail = [];
                    var list;
                    var baselineMean = baseline.stats.mean;
                    var thresholds = _this.thresholds || {};
                    var percentDifference = (100 * (benchmark.stats.mean - baselineMean)) / baselineMean;
                    if (thresholds.warn &&
                        thresholds.warn.mean &&
                        Math.abs(percentDifference) > thresholds.warn.mean) {
                        list = warn;
                        if (thresholds.fail &&
                            thresholds.fail.mean &&
                            Math.abs(percentDifference) > thresholds.fail.mean) {
                            list = fail;
                        }
                        list.push('Execution time is ' + percentDifference.toFixed(1) + '% off');
                    }
                    var baselineRme = baseline.stats.rme;
                    percentDifference = benchmark.stats.rme - baselineRme;
                    if (thresholds.warn &&
                        thresholds.warn.rme &&
                        Math.abs(percentDifference) > thresholds.warn.rme) {
                        list = warn;
                        if (thresholds.fail &&
                            thresholds.fail.rme &&
                            Math.abs(percentDifference) > thresholds.fail.rme) {
                            list = fail;
                        }
                        list.push('RME is ' + percentDifference.toFixed(1) + '% off');
                    }
                    if (fail.length) {
                        _this.console.error('FAIL ' + benchmarkTest.id + ' (' + fail.join(', ') + ')');
                        return false;
                    }
                    else if (warn.length) {
                        _this.console.warn('WARN ' + benchmarkTest.id + ' (' + warn.join(', ') + ')');
                    }
                    else {
                        _this.console.log('PASS ' + benchmarkTest.id);
                    }
                    return true;
                };
                var benchmark = benchmarkTest.benchmark;
                var session = this._getSession(benchmarkTest);
                var environment = session.environment;
                var suiteInfo = session.suites[benchmarkTest.parentId];
                suiteInfo.numBenchmarks++;
                var baseline = this.baseline[environment.id];
                if (this.mode === 'baseline') {
                    baseline.tests[benchmarkTest.id] = {
                        hz: benchmark.hz,
                        times: benchmark.times,
                        stats: {
                            rme: benchmark.stats.rme,
                            moe: benchmark.stats.moe,
                            mean: benchmark.stats.mean
                        }
                    };
                    this.console.log('Baselined ' + benchmarkTest.name);
                    this.executor.log('Time per run:', formatSeconds(benchmark.stats.mean), '\xb1', benchmark.stats.rme.toFixed(2), '%');
                }
                else {
                    if (baseline) {
                        var testData = baseline.tests[benchmarkTest.id];
                        var result = checkTest(testData, benchmark);
                        var baselineStats = testData.stats;
                        var benchmarkStats = benchmark.stats;
                        this.executor.log('Expected time per run:', formatSeconds(baselineStats.mean), '\xb1', baselineStats.rme.toFixed(2), '%');
                        this.executor.log('Actual time per run:', formatSeconds(benchmarkStats.mean), '\xb1', benchmarkStats.rme.toFixed(2), '%');
                        if (!result) {
                            suiteInfo.numFailedBenchmarks++;
                        }
                    }
                }
            }
        };
        tslib_1.__decorate([
            Reporter_1.eventHandler()
        ], BenchmarkReporter.prototype, "runEnd", null);
        tslib_1.__decorate([
            Reporter_1.eventHandler()
        ], BenchmarkReporter.prototype, "suiteEnd", null);
        tslib_1.__decorate([
            Reporter_1.eventHandler()
        ], BenchmarkReporter.prototype, "suiteStart", null);
        tslib_1.__decorate([
            Reporter_1.eventHandler()
        ], BenchmarkReporter.prototype, "testEnd", null);
        return BenchmarkReporter;
    }(Reporter_1.default));
    exports.default = BenchmarkReporter;
    function formatSeconds(value) {
        if (value == null) {
            return null;
        }
        var units = 's';
        if (value < 1) {
            var places = Math.ceil(Math.log(value) / Math.log(10)) - 1;
            if (places < -9) {
                value *= Math.pow(10, 12);
                units = 'ps';
            }
            else if (places < -6) {
                value *= Math.pow(10, 9);
                units = 'ns';
            }
            else if (places < -3) {
                value *= Math.pow(10, 6);
                units = 'µs';
            }
            else if (places < 0) {
                value *= Math.pow(10, 3);
                units = 'ms';
            }
        }
        return value.toFixed(3) + units;
    }
});
//# sourceMappingURL=Benchmark.js.map