tachometer/lib/runner.js

"use strict";
/**
 * @license
 * Copyright (c) 2019 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at
 * http://polymer.github.io/LICENSE.txt The complete set of authors may be found
 * at http://polymer.github.io/AUTHORS.txt The complete set of contributors may
 * be found at http://polymer.github.io/CONTRIBUTORS.txt Code distributed by
 * Google as part of the polymer project is also subject to an additional IP
 * rights grant found at http://polymer.github.io/PATENTS.txt
 */
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Runner = void 0;
const fsExtra = __importStar(require("fs-extra"));
const ProgressBar = require("progress");
const ansi = require("ansi-escape-sequences");
const json_output_1 = require("./json-output");
const browser_1 = require("./browser");
const measure_1 = require("./measure");
const csv_1 = require("./csv");
const stats_1 = require("./stats");
const format_1 = require("./format");
const github = __importStar(require("./github"));
const specs_1 = require("./specs");
const util_1 = require("./util");
class Runner {
    constructor(config, servers) {
        this.browsers = new Map();
        this.results = new Map();
        /**
         * How many times we will load a page and try to collect all measurements
         * before fully failing.
         */
        this.maxAttempts = 3;
        /**
         * Maximum milliseconds we will wait for all measurements to be collected per
         * attempt before reloading and trying a new attempt.
         */
        this.attemptTimeout = 10000;
        /**
         * How many milliseconds we will wait between each poll for measurements.
         */
        this.pollTime = 50;
        this.hitTimeout = false;
        this.config = config;
        this.specs = config.benchmarks;
        this.servers = servers;
        this.bar = new ProgressBar('[:bar] :status', {
            total: this.specs.length * (config.sampleSize + /** warmup */ 1),
            width: 58,
        });
    }
    async run() {
        await this.launchBrowsers();
        if (this.config.githubCheck !== undefined) {
            this.completeGithubCheck =
                await github.createCheck(this.config.githubCheck);
        }
        console.log('Running benchmarks\n');
        await this.warmup();
        await this.takeMinimumSamples();
        await this.takeAdditionalSamples();
        await this.closeBrowsers();
        const results = this.makeResults();
        await this.outputResults(results);
        return results;
    }
    async launchBrowsers() {
        for (const { browser } of this.specs) {
            const sig = browser_1.browserSignature(browser);
            if (this.browsers.has(sig)) {
                continue;
            }
            this.bar.tick(0, { status: `launching ${browser.name}` });
            // It's important that we execute each benchmark iteration in a new tab.
            // At least in Chrome, each tab corresponds to process which shares some
            // amount of cached V8 state which can cause significant measurement
            // effects. There might even be additional interaction effects that
            // would require an entirely new browser to remove, but experience in
            // Chrome so far shows that new tabs are neccessary and sufficient.
            const driver = await browser_1.makeDriver(browser);
            const tabs = await driver.getAllWindowHandles();
            // We'll always launch new tabs from this initial blank tab.
            const initialTabHandle = tabs[0];
            this.browsers.set(sig, { name: browser.name, driver, initialTabHandle });
        }
    }
    async closeBrowsers() {
        // Close the browsers by closing each of their last remaining tabs.
        await Promise.all([...this.browsers.values()].map(({ driver }) => driver.close()));
    }
    /**
     * Do one throw-away run per benchmark to warm up our server (especially
     * when expensive bare module resolution is enabled), and the browser.
     */
    async warmup() {
        const { specs, bar } = this;
        for (let i = 0; i < specs.length; i++) {
            const spec = specs[i];
            bar.tick(0, {
                status: `warmup ${i + 1}/${specs.length} ${format_1.benchmarkOneLiner(spec)}`,
            });
            await this.takeSamples(spec);
            bar.tick(1);
        }
    }
    recordSamples(spec, newResults) {
        let specResults = this.results.get(spec);
        if (specResults === undefined) {
            specResults = [];
            this.results.set(spec, specResults);
        }
        // This function is called once per page per sample. The first time this
        // function is called for a page, that result object becomes our "primary"
        // one. On subsequent calls, we accrete the additional sample data into this
        // primary one. The other fields are always the same, so we can just ignore
        // them after the first call.
        // TODO(aomarks) The other fields (user agent, bytes sent, etc.) only need
        // to be collected on the first run of each page, so we could do that in the
        // warmup phase, and then function would only need to take sample data,
        // since it's a bit confusing how we throw away a bunch of fields after the
        // first call.
        for (const newResult of newResults) {
            const primary = specResults[newResult.measurementIndex];
            if (primary === undefined) {
                specResults[newResult.measurementIndex] = newResult;
            }
            else {
                primary.millis.push(...newResult.millis);
            }
        }
    }
    async takeMinimumSamples() {
        // Always collect our minimum number of samples.
        const { config, specs, bar } = this;
        const numRuns = specs.length * config.sampleSize;
        let run = 0;
        for (let sample = 0; sample < config.sampleSize; sample++) {
            for (const spec of specs) {
                bar.tick(0, {
                    status: `${++run}/${numRuns} ${format_1.benchmarkOneLiner(spec)}`,
                });
                this.recordSamples(spec, await this.takeSamples(spec));
                if (bar.curr === bar.total - 1) {
                    // Note if we tick with 0 after we've completed, the status is
                    // rendered on the next line for some reason.
                    bar.tick(1, { status: 'done' });
                }
                else {
                    bar.tick(1);
                }
            }
        }
    }
    async takeAdditionalSamples() {
        const { config, specs } = this;
        if (config.timeout <= 0) {
            return;
        }
        console.log();
        const timeoutMs = config.timeout * 60 * 1000; // minutes -> millis
        const startMs = Date.now();
        let run = 0;
        let sample = 0;
        let elapsed = 0;
        while (true) {
            if (stats_1.horizonsResolved(this.makeResults(), config.horizons)) {
                console.log();
                break;
            }
            if (elapsed >= timeoutMs) {
                this.hitTimeout = true;
                break;
            }
            // Run batches of 10 additional samples at a time for more presentable
            // sample sizes, and to nudge sample sizes up a little.
            for (let i = 0; i < 10; i++) {
                sample++;
                for (const spec of specs) {
                    run++;
                    elapsed = Date.now() - startMs;
                    const remainingSecs = Math.max(0, Math.round((timeoutMs - elapsed) / 1000));
                    const mins = Math.floor(remainingSecs / 60);
                    const secs = remainingSecs % 60;
                    process.stdout.write(`\r${format_1.spinner[run % format_1.spinner.length]} Auto-sample ${sample} ` +
                        `(timeout in ${mins}m${secs}s)` + ansi.erase.inLine(0));
                    this.recordSamples(spec, await this.takeSamples(spec));
                }
            }
        }
    }
    async takeSamples(spec) {
        const { servers, config, browsers } = this;
        let server;
        if (spec.url.kind === 'local') {
            server = servers.get(spec);
            if (server === undefined) {
                throw new Error('Internal error: no server for spec');
            }
        }
        const url = specs_1.specUrl(spec, servers, config);
        const { driver, initialTabHandle } = browsers.get(browser_1.browserSignature(spec.browser));
        let session;
        let pendingMeasurements;
        let measurementResults;
        // We'll try N attempts per page. Within each attempt, we'll try to collect
        // all of the measurements by polling. If we hit our per-attempt timeout
        // before collecting all measurements, we'll move onto the next attempt
        // where we reload the whole page and start from scratch. If we hit our max
        // attempts, we'll throw.
        for (let pageAttempt = 1;; pageAttempt++) {
            // New attempt. Reset all measurements and results.
            pendingMeasurements = new Set(spec.measurement);
            measurementResults = [];
            await browser_1.openAndSwitchToNewTab(driver, spec.browser);
            await driver.get(url);
            for (let waited = 0; pendingMeasurements.size > 0 && waited <= this.attemptTimeout; waited += this.pollTime) {
                // TODO(aomarks) You don't have to wait in callback mode!
                await util_1.wait(this.pollTime);
                for (let measurementIndex = 0; measurementIndex < spec.measurement.length; measurementIndex++) {
                    if (measurementResults[measurementIndex] !== undefined) {
                        // Already collected this measurement on this attempt.
                        continue;
                    }
                    const measurement = spec.measurement[measurementIndex];
                    const result = await measure_1.measure(driver, measurement, server);
                    if (result !== undefined) {
                        measurementResults[measurementIndex] = result;
                        pendingMeasurements.delete(measurement);
                    }
                }
            }
            // Close the active tab (but not the whole browser, since the
            // initial blank tab is still open).
            await driver.close();
            await driver.switchTo().window(initialTabHandle);
            if (server !== undefined) {
                session = server.endSession();
            }
            if (pendingMeasurements.size === 0 || pageAttempt >= this.maxAttempts) {
                break;
            }
            console.log(`\n\nFailed ${pageAttempt}/${this.maxAttempts} times ` +
                `to get measurement(s) ${spec.name}` +
                (spec.measurement.length > 1 ? ` [${[...pendingMeasurements]
                    .map(measure_1.measurementName)
                    .join(', ')}]` :
                    '') +
                ` in ${spec.browser.name} from ${url}. Retrying.`);
        }
        if (pendingMeasurements.size > 0) {
            console.log();
            throw new Error(`\n\nFailed ${this.maxAttempts}/${this.maxAttempts} times ` +
                `to get measurement(s) ${spec.name}` +
                (spec.measurement.length > 1 ? ` [${[...pendingMeasurements]
                    .map(measure_1.measurementName)
                    .join(', ')}]` :
                    '') +
                ` in ${spec.browser.name} from ${url}`);
        }
        return spec.measurement.map((measurement, measurementIndex) => ({
            name: spec.measurement.length === 1 ?
                spec.name :
                `${spec.name} [${measure_1.measurementName(measurement)}]`,
            measurement,
            measurementIndex: measurementIndex,
            queryString: spec.url.kind === 'local' ? spec.url.queryString : '',
            version: spec.url.kind === 'local' && spec.url.version !== undefined ?
                spec.url.version.label :
                '',
            millis: [measurementResults[measurementIndex]],
            bytesSent: session ? session.bytesSent : 0,
            browser: spec.browser,
            userAgent: session ? session.userAgent : '',
        }));
    }
    makeResults() {
        const resultStats = [];
        for (const results of this.results.values()) {
            for (let r = 0; r < results.length; r++) {
                const result = results[r];
                resultStats.push({ result, stats: stats_1.summaryStats(result.millis) });
            }
        }
        return stats_1.computeDifferences(resultStats);
    }
    async outputResults(withDifferences) {
        const { config, hitTimeout } = this;
        console.log();
        const { fixed, unfixed } = format_1.automaticResultTable(withDifferences);
        console.log(format_1.horizontalTermResultTable(fixed));
        console.log(format_1.verticalTermResultTable(unfixed));
        if (hitTimeout === true) {
            console.log(ansi.format(`[bold red]{NOTE} Hit ${config.timeout} minute auto-sample timeout` +
                ` trying to resolve horizon(s)`));
            console.log('Consider a longer --timeout or different --horizon');
        }
        if (config.jsonFile) {
            const json = await json_output_1.jsonOutput(withDifferences);
            await fsExtra.writeJSON(config.jsonFile, json, { spaces: 2 });
        }
        // TOOD(aomarks) Remove this in next major version.
        if (config.legacyJsonFile) {
            const json = await json_output_1.legacyJsonOutput(withDifferences.map((s) => s.result));
            await fsExtra.writeJSON(config.legacyJsonFile, json);
        }
        if (config.csvFileStats) {
            await fsExtra.writeFile(config.csvFileStats, csv_1.formatCsvStats(withDifferences));
        }
        if (config.csvFileRaw) {
            await fsExtra.writeFile(config.csvFileRaw, csv_1.formatCsvRaw(withDifferences));
        }
        if (this.completeGithubCheck !== undefined) {
            const markdown = format_1.horizontalHtmlResultTable(fixed) + '\n' +
                format_1.verticalHtmlResultTable(unfixed);
            await this.completeGithubCheck(markdown);
        }
    }
}
exports.Runner = Runner;
//# sourceMappingURL=runner.js.map