| 1 | {"version":3,"file":"Benchmark.js","sourceRoot":"","sources":["../../../../src/lib/reporters/Benchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;IAAA,yBAAiD;IAGjD,6DAAwE;IA6BxE;QAA+C,6CAAQ;QAYrD,2BAAY,QAAkB,EAAE,OAAsC;YAAtC,wBAAA,EAAA,YAAsC;YAAtE,YACE,kBAAM,QAAQ,EAAE,OAAO,CAAC,SA4BzB;YA1BC,KAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC;YACnC,KAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;YACvC,KAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;YAG3C,IAAI,KAAI,CAAC,IAAI,KAAK,MAAM,EAAE;gBACxB,IAAI;oBACF,KAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CACxB,iBAAY,CAAC,KAAI,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAClD,CAAC;iBACH;gBAAC,OAAO,KAAK,EAAE;oBACd,KAAI,CAAC,OAAO,CAAC,IAAI,CACf,8CAA8C,GAAG,KAAI,CAAC,QAAQ,CAC/D,CAAC;oBACF,KAAI,CAAC,OAAO,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;oBAClD,KAAI,CAAC,IAAI,GAAG,UAAU,CAAC;iBACxB;aACF;YAED,IAAI,CAAC,KAAI,CAAC,QAAQ,EAAE;gBAClB,KAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;aACpB;YAID,KAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;;QACrB,CAAC;QAED,uCAAW,GAAX,UAAY,WAAyB;YACnC,IAAM,SAAS,GAAG,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC;YACnD,IAAI,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAEvC,IAAI,CAAC,OAAO,EAAE;gBACZ,IAAI,MAAM,SAAQ,CAAC;gBACnB,IAAI,OAAO,SAAQ,CAAC;gBACpB,IAAI,UAAgB,CAAC;gBAErB,IAAI,WAAW,CAAC,SAAS,EAAE;oBACzB,IAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,eAAgB,CAAC;oBAC5D,MAAM,GAAG,eAAe,CAAC,WAAY,CAAC;oBACtC,OAAO,GAAG,eAAe,CAAC,OAAQ,CAAC;oBACnC,UAAQ,GAAG,eAAe,CAAC,QAAS,CAAC;iBACtC;qBAAM;oBACL,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;oBACvB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;oBAC1B,UAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;iBAC7B;gBAED,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG;oBACnC,MAAM,EAAE,EAAE;oBACV,WAAW,EAAE;wBACX,MAAM,QAAA;wBACN,OAAO,SAAA;wBACP,QAAQ,YAAA;wBACR,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,OAAO,GAAG,GAAG,GAAG,UAAQ;qBAC5C;iBACF,CAAC;aACH;YAED,OAAO,OAAO,CAAC;QACjB,CAAC;QAGD,kCAAM,GAAN;YACE,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE;gBAC5B,IAAI,kBAAmC,CAAC;gBACxC,IAAI;oBACF,kBAAgB,GAAG,IAAI,CAAC,KAAK,CAC3B,iBAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAClD,CAAC;iBACH;gBAAC,OAAO,KAAK,EAAE;oBACd,kBAAgB,GAAG,EAAE,CAAC;iBACvB;gBAID,IAAM,UAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,UAAQ,CAAC,CAAC,OAAO,CAAC,UAAS,aAAa;oBAClD,kBAAgB,CAAC,aAAa,CAAC,GAAG,UAAQ,CAAC,aAAa,CAAC,CAAC;gBAC5D,CAAC,CAAC,CAAC;gBACH,kBAAa,CACX,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,SAAS,CAAC,kBAAgB,EAAE,IAAI,EAAE,MAAM,CAAC,CAC/C,CAAC;aACH;QACH,CAAC;QAGD,oCAAQ,GAAR,UAAS,KAAY;YACnB,IAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAExC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;gBACpB,IAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;gBACxC,IAAI,CAAC,OAAO,CAAC,GAAG,CACd,wBAAwB;oBACtB,WAAW,CAAC,MAAM;oBAClB,GAAG;oBACH,WAAW,CAAC,OAAO;oBACnB,MAAM;oBACN,WAAW,CAAC,QAAQ,CACvB,CAAC;aACH;iBAAM,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE;gBAC/B,IAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAC3C,IAAM,QAAQ,GAAG,SAAS,CAAC,aAAa,CAAC;gBACzC,IAAI,QAAQ,GAAG,CAAC,EAAE;oBAChB,IAAM,cAAc,GAAG,SAAS,CAAC,mBAAmB,CAAC;oBACrD,IAAM,OAAO,GACX,cAAc,GAAG,GAAG,GAAG,QAAQ,GAAG,wBAAwB,GAAG,KAAK,CAAC,EAAE,CAAC;oBACxE,IAAI,cAAc,GAAG,CAAC,EAAE;wBACtB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;qBAC5B;yBAAM;wBACL,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;qBAC3B;iBACF;aACF;QACH,CAAC;QAGD,sCAAU,GAAV,UAAW,KAAY;YACrB,IAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAGxC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;gBACpB,IAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;gBACxC,IAAM,eAAe,GACnB,WAAW,CAAC,MAAM;oBAClB,GAAG;oBACH,WAAW,CAAC,OAAO;oBACnB,MAAM;oBACN,WAAW,CAAC,QAAQ,CAAC;gBACvB,IAAM,oBAAoB,GAAG,IAAI,CAAC,QAAQ,CAAC;gBAE3C,IAAI,CAAC,OAAO,CAAC,GAAG,CACd,CAAC,IAAI,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC;oBACxD,GAAG;oBACH,eAAe,CAClB,CAAC;gBAEF,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE;oBAC5B,oBAAoB,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG;wBACrC,MAAM,EAAE,WAAW,CAAC,MAAM;wBAC1B,OAAO,EAAE,WAAW,CAAC,OAAO;wBAC5B,QAAQ,EAAE,WAAW,CAAC,QAAQ;wBAC9B,KAAK,EAAE,EAAE;qBACV,CAAC;iBACH;qBAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE;oBAChD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,uBAAuB,GAAG,eAAe,GAAG,GAAG,CAAC,CAAC;iBACpE;aACF;iBAAM;gBACL,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG;oBACzB,aAAa,EAAE,CAAC;oBAChB,mBAAmB,EAAE,CAAC;iBACvB,CAAC;aACH;QACH,CAAC;QAGD,mCAAO,GAAP,UAAQ,IAAU;YADlB,iBA0IC;YAxIC,IAAM,aAAa,GAAkB,IAAI,CAAC;YAI1C,IAAI,aAAa,CAAC,SAAS,IAAI,IAAI,EAAE;gBACnC,OAAO;aACR;YAED,IAAI,aAAa,CAAC,KAAK,EAAE;gBACvB,IAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;gBAChD,IAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;gBACzD,SAAS,CAAC,aAAa,EAAE,CAAC;gBAC1B,SAAS,CAAC,mBAAmB,EAAE,CAAC;gBAEhC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;gBAChD,IAAI,CAAC,OAAO,CAAC,KAAK,CAChB,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,aAAa,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAChE,CAAC;aACH;iBAAM;gBACL,IAAM,SAAS,GAAG,UAAC,QAAuB,EAAE,SAAwB;oBAClE,IAAI,IAAI,GAAa,EAAE,CAAC;oBACxB,IAAI,IAAI,GAAa,EAAE,CAAC;oBACxB,IAAI,IAAc,CAAC;oBAEnB,IAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;oBACzC,IAAM,UAAU,GAAG,KAAI,CAAC,UAAU,IAAI,EAAE,CAAC;oBACzC,IAAI,iBAAiB,GACnB,CAAC,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,GAAG,YAAY,CAAC,CAAC,GAAG,YAAY,CAAC;oBAE/D,IACE,UAAU,CAAC,IAAI;wBACf,UAAU,CAAC,IAAI,CAAC,IAAI;wBACpB,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAClD;wBACA,IAAI,GAAG,IAAI,CAAC;wBACZ,IACE,UAAU,CAAC,IAAI;4BACf,UAAU,CAAC,IAAI,CAAC,IAAI;4BACpB,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,EAClD;4BACA,IAAI,GAAG,IAAI,CAAC;yBACb;wBACD,IAAI,CAAC,IAAI,CACP,oBAAoB,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAC9D,CAAC;qBACH;oBAED,IAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC;oBAEvC,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,GAAG,WAAW,CAAC;oBACtD,IACE,UAAU,CAAC,IAAI;wBACf,UAAU,CAAC,IAAI,CAAC,GAAG;wBACnB,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,EACjD;wBACA,IAAI,GAAG,IAAI,CAAC;wBACZ,IACE,UAAU,CAAC,IAAI;4BACf,UAAU,CAAC,IAAI,CAAC,GAAG;4BACnB,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,EACjD;4BACA,IAAI,GAAG,IAAI,CAAC;yBACb;wBACD,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;qBAC/D;oBAED,IAAI,IAAI,CAAC,MAAM,EAAE;wBACf,KAAI,CAAC,OAAO,CAAC,KAAK,CAChB,OAAO,GAAG,aAAa,CAAC,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAC1D,CAAC;wBACF,OAAO,KAAK,CAAC;qBACd;yBAAM,IAAI,IAAI,CAAC,MAAM,EAAE;wBACtB,KAAI,CAAC,OAAO,CAAC,IAAI,CACf,OAAO,GAAG,aAAa,CAAC,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAC1D,CAAC;qBACH;yBAAM;wBACL,KAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;qBAC9C;oBAED,OAAO,IAAI,CAAC;gBACd,CAAC,CAAC;gBAEF,IAAM,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;gBAC1C,IAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;gBAChD,IAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;gBAExC,IAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;gBACzD,SAAS,CAAC,aAAa,EAAE,CAAC;gBAE1B,IAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAE,CAAC;gBAEhD,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE;oBAC5B,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,GAAG;wBACjC,EAAE,EAAE,SAAS,CAAC,EAAE;wBAChB,KAAK,EAAE,SAAS,CAAC,KAAK;wBACtB,KAAK,EAAE;4BACL,GAAG,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG;4BACxB,GAAG,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG;4BACxB,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,IAAI;yBAC3B;qBACF,CAAC;oBACF,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;oBACpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,eAAe,EACf,aAAa,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EACnC,MAAM,EACN,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAC9B,GAAG,CACJ,CAAC;iBACH;qBAAM;oBACL,IAAI,QAAQ,EAAE;wBACZ,IAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;wBAClD,IAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;wBAC9C,IAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC;wBACrC,IAAM,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC;wBACvC,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,wBAAwB,EACxB,aAAa,CAAC,aAAa,CAAC,IAAI,CAAC,EACjC,MAAM,EACN,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAC5B,GAAG,CACJ,CAAC;wBACF,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,sBAAsB,EACtB,aAAa,CAAC,cAAc,CAAC,IAAI,CAAC,EAClC,MAAM,EACN,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAC7B,GAAG,CACJ,CAAC;wBAEF,IAAI,CAAC,MAAM,EAAE;4BACX,SAAS,CAAC,mBAAmB,EAAE,CAAC;yBACjC;qBACF;iBACF;aACF;QACH,CAAC;QAvOD;YADC,uBAAY,EAAE;uDAuBd;QAGD;YADC,uBAAY,EAAE;yDA4Bd;QAGD;YADC,uBAAY,EAAE;2DAqCd;QAGD;YADC,uBAAY,EAAE;wDA0Id;QACH,wBAAC;KAAA,AAtTD,CAA+C,kBAAQ,GAsTtD;sBAtToB,iBAAiB;IAqXtC,SAAS,aAAa,CAAC,KAAa;QAClC,IAAI,KAAK,IAAI,IAAI,EAAE;YACjB,OAAO,IAAI,CAAC;SACb;QAED,IAAI,KAAK,GAAG,GAAG,CAAC;QAChB,IAAI,KAAK,GAAG,CAAC,EAAE;YACb,IAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,MAAM,GAAG,CAAC,CAAC,EAAE;gBACf,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC1B,KAAK,GAAG,IAAI,CAAC;aACd;iBAAM,IAAI,MAAM,GAAG,CAAC,CAAC,EAAE;gBACtB,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBACzB,KAAK,GAAG,IAAI,CAAC;aACd;iBAAM,IAAI,MAAM,GAAG,CAAC,CAAC,EAAE;gBACtB,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBACzB,KAAK,GAAG,IAAI,CAAC;aACd;iBAAM,IAAI,MAAM,GAAG,CAAC,EAAE;gBACrB,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBACzB,KAAK,GAAG,IAAI,CAAC;aACd;SACF;QAED,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IAClC,CAAC","sourcesContent":["import { readFileSync, writeFileSync } from 'fs';\nimport * as Benchmark from 'benchmark';\nimport { Executor } from '../executors/Executor';\nimport Reporter, { eventHandler, ReporterProperties } from './Reporter';\nimport BenchmarkTest from '../BenchmarkTest';\nimport Test from '../Test';\nimport Suite from '../Suite';\n\n/**\n * Benchmark is a reporter that can generate a baseline report and do runtime\n * comparisons against an existing baseline.\n *\n * **Configuration**\n *\n * Along with the default reporter options, Benchmark also supports a `mode`\n * option. This can have two values:\n *\n * * `'baseline'`: Benchmark data will be written to a baseline file when\n * testing is finished\n * * `'test'`: Benchmark is compared to a baseline read from a file when testing\n * starts\n *\n * Baseline data is stored hierarchically by environment and then by test.\n *\n * **Notation**\n *\n * * **rme:** relative margin of error -- margin of error as a percentage of the\n * mean margin of error\n * * **mean:** mean execution time per function run\n * * **hz:** Hertz (number of executions of a function per second). 1/Hz is the\n * mean execution time of function.\n */\nexport default class BenchmarkReporter extends Reporter\n implements BenchmarkReporterProperties {\n baseline!: BenchmarkBaseline;\n\n filename: string;\n\n mode: BenchmarkMode;\n\n sessions: { [sessionId: string]: SessionInfo };\n\n thresholds: BenchmarkThresholds;\n\n constructor(executor: Executor, options: BenchmarkReporterOptions = {}) {\n super(executor, options);\n\n this.mode = options.mode || 'test';\n this.filename = options.filename || '';\n this.thresholds = options.thresholds || {};\n\n // In test mode, try to load benchmark data for comparison\n if (this.mode === 'test') {\n try {\n this.baseline = JSON.parse(\n readFileSync(this.filename, { encoding: 'utf8' })\n );\n } catch (error) {\n this.console.warn(\n 'Unable to load benchmark baseline data from ' + this.filename\n );\n this.console.warn('Switching to \"baseline\" mode');\n this.mode = 'baseline';\n }\n }\n\n if (!this.baseline) {\n this.baseline = {};\n }\n\n // Cache environments by session ID so we can look them up again when\n // serialized tests come back from remote browsers\n this.sessions = {};\n }\n\n _getSession(testOrSuite: Test | Suite) {\n const sessionId = testOrSuite.sessionId || 'local';\n let session = this.sessions[sessionId];\n\n if (!session) {\n let client: string;\n let version: string;\n let platform: string;\n\n if (testOrSuite.sessionId) {\n const environmentType = testOrSuite.remote.environmentType!;\n client = environmentType.browserName!;\n version = environmentType.version!;\n platform = environmentType.platform!;\n } else {\n client = process.title;\n version = process.version;\n platform = process.platform;\n }\n\n session = this.sessions[sessionId] = {\n suites: {},\n environment: {\n client,\n version,\n platform,\n id: client + ':' + version + ':' + platform\n }\n };\n }\n\n return session;\n }\n\n @eventHandler()\n runEnd() {\n if (this.mode === 'baseline') {\n let existingBaseline: BenchmarkBaseline;\n try {\n existingBaseline = JSON.parse(\n readFileSync(this.filename, { encoding: 'utf8' })\n );\n } catch (error) {\n existingBaseline = {};\n }\n\n // Merge the newly recorded baseline data into the existing baseline\n // data and write it back out to output file.\n const baseline = this.baseline;\n Object.keys(baseline).forEach(function(environmentId) {\n existingBaseline[environmentId] = baseline[environmentId];\n });\n writeFileSync(\n this.filename,\n JSON.stringify(existingBaseline, null, ' ')\n );\n }\n }\n\n @eventHandler()\n suiteEnd(suite: Suite) {\n const session = this._getSession(suite);\n\n if (!suite.hasParent) {\n const environment = session.environment;\n this.console.log(\n 'Finished benchmarking ' +\n environment.client +\n ' ' +\n environment.version +\n ' on ' +\n environment.platform\n );\n } else if (this.mode === 'test') {\n const suiteInfo = session.suites[suite.id];\n const numTests = suiteInfo.numBenchmarks;\n if (numTests > 0) {\n const numFailedTests = suiteInfo.numFailedBenchmarks;\n const message =\n numFailedTests + '/' + numTests + ' benchmarks failed in ' + suite.id;\n if (numFailedTests > 0) {\n this.console.warn(message);\n } else {\n this.console.log(message);\n }\n }\n }\n }\n\n @eventHandler()\n suiteStart(suite: Suite) {\n const session = this._getSession(suite);\n\n // This is a session root suite\n if (!suite.hasParent) {\n const environment = session.environment;\n const environmentName =\n environment.client +\n ' ' +\n environment.version +\n ' on ' +\n environment.platform;\n const baselineEnvironments = this.baseline;\n\n this.console.log(\n (this.mode === 'baseline' ? 'Baselining' : 'Benchmarking') +\n ' ' +\n environmentName\n );\n\n if (this.mode === 'baseline') {\n baselineEnvironments[environment.id] = {\n client: environment.client,\n version: environment.version,\n platform: environment.platform,\n tests: {}\n };\n } else if (!baselineEnvironments[environment.id]) {\n this.console.warn('No baseline data for ' + environmentName + '!');\n }\n } else {\n session.suites[suite.id] = {\n numBenchmarks: 0,\n numFailedBenchmarks: 0\n };\n }\n }\n\n @eventHandler()\n testEnd(test: Test) {\n const benchmarkTest = <BenchmarkTest>test;\n\n // Just check for the benchmark property because the test may be a\n // deserialized object rather than an actual BenchmarkTest instance.\n if (benchmarkTest.benchmark == null) {\n return;\n }\n\n if (benchmarkTest.error) {\n const session = this._getSession(benchmarkTest);\n const suiteInfo = session.suites[benchmarkTest.parentId];\n suiteInfo.numBenchmarks++;\n suiteInfo.numFailedBenchmarks++;\n\n this.console.error('FAIL: ' + benchmarkTest.id);\n this.console.error(\n this.executor.formatError(benchmarkTest.error, { space: ' ' })\n );\n } else {\n const checkTest = (baseline: BenchmarkData, benchmark: BenchmarkData) => {\n let warn: string[] = [];\n let fail: string[] = [];\n let list: string[];\n\n const baselineMean = baseline.stats.mean;\n const thresholds = this.thresholds || {};\n let percentDifference =\n (100 * (benchmark.stats.mean - baselineMean)) / baselineMean;\n\n if (\n thresholds.warn &&\n thresholds.warn.mean &&\n Math.abs(percentDifference) > thresholds.warn.mean\n ) {\n list = warn;\n if (\n thresholds.fail &&\n thresholds.fail.mean &&\n Math.abs(percentDifference) > thresholds.fail.mean\n ) {\n list = fail;\n }\n list.push(\n 'Execution time is ' + percentDifference.toFixed(1) + '% off'\n );\n }\n\n const baselineRme = baseline.stats.rme;\n // RME is already a percent\n percentDifference = benchmark.stats.rme - baselineRme;\n if (\n thresholds.warn &&\n thresholds.warn.rme &&\n Math.abs(percentDifference) > thresholds.warn.rme\n ) {\n list = warn;\n if (\n thresholds.fail &&\n thresholds.fail.rme &&\n Math.abs(percentDifference) > thresholds.fail.rme\n ) {\n list = fail;\n }\n list.push('RME is ' + percentDifference.toFixed(1) + '% off');\n }\n\n if (fail.length) {\n this.console.error(\n 'FAIL ' + benchmarkTest.id + ' (' + fail.join(', ') + ')'\n );\n return false;\n } else if (warn.length) {\n this.console.warn(\n 'WARN ' + benchmarkTest.id + ' (' + warn.join(', ') + ')'\n );\n } else {\n this.console.log('PASS ' + benchmarkTest.id);\n }\n\n return true;\n };\n\n const benchmark = benchmarkTest.benchmark;\n const session = this._getSession(benchmarkTest);\n const environment = session.environment;\n\n const suiteInfo = session.suites[benchmarkTest.parentId];\n suiteInfo.numBenchmarks++;\n\n const baseline = this.baseline[environment.id]!;\n\n if (this.mode === 'baseline') {\n baseline.tests[benchmarkTest.id] = {\n hz: benchmark.hz,\n times: benchmark.times,\n stats: {\n rme: benchmark.stats.rme,\n moe: benchmark.stats.moe,\n mean: benchmark.stats.mean\n }\n };\n this.console.log('Baselined ' + benchmarkTest.name);\n this.executor.log(\n 'Time per run:',\n formatSeconds(benchmark.stats.mean),\n '\\xb1',\n benchmark.stats.rme.toFixed(2),\n '%'\n );\n } else {\n if (baseline) {\n const testData = baseline.tests[benchmarkTest.id];\n const result = checkTest(testData, benchmark);\n const baselineStats = testData.stats;\n const benchmarkStats = benchmark.stats;\n this.executor.log(\n 'Expected time per run:',\n formatSeconds(baselineStats.mean),\n '\\xb1',\n baselineStats.rme.toFixed(2),\n '%'\n );\n this.executor.log(\n 'Actual time per run:',\n formatSeconds(benchmarkStats.mean),\n '\\xb1',\n benchmarkStats.rme.toFixed(2),\n '%'\n );\n\n if (!result) {\n suiteInfo.numFailedBenchmarks++;\n }\n }\n }\n }\n }\n}\n\nexport interface BenchmarkData {\n times: Benchmark.Times;\n hz: number;\n stats: {\n rme: number;\n moe: number;\n mean: number;\n };\n}\n\nexport interface BenchmarkThresholds {\n warn?: {\n rme?: number;\n hz?: number;\n mean?: number;\n };\n fail?: {\n rme?: number;\n hz?: number;\n mean?: number;\n };\n}\n\nexport interface BaselineEnvironment {\n client: string;\n version: string;\n platform: string;\n tests: { [testId: string]: BenchmarkData };\n}\n\nexport interface BenchmarkBaseline {\n [key: string]: BaselineEnvironment;\n}\n\nexport interface SesssionEnvironment {\n client: string;\n version: string;\n platform: string;\n id: string;\n}\n\nexport interface SessionInfo {\n suites: {\n [suiteId: string]: {\n numBenchmarks: number;\n numFailedBenchmarks: number;\n };\n };\n environment: SesssionEnvironment;\n}\n\nexport type BenchmarkMode = 'baseline' | 'test';\n\nexport interface BenchmarkReporterProperties extends ReporterProperties {\n filename: string;\n mode: BenchmarkMode;\n thresholds: BenchmarkThresholds;\n}\n\nexport type BenchmarkReporterOptions = Partial<BenchmarkReporterProperties>;\n\nfunction formatSeconds(value: number) {\n if (value == null) {\n return null;\n }\n\n let units = 's';\n if (value < 1) {\n const places = Math.ceil(Math.log(value) / Math.log(10)) - 1;\n if (places < -9) {\n value *= Math.pow(10, 12);\n units = 'ps';\n } else if (places < -6) {\n value *= Math.pow(10, 9);\n units = 'ns';\n } else if (places < -3) {\n value *= Math.pow(10, 6);\n units = 'µs';\n } else if (places < 0) {\n value *= Math.pow(10, 3);\n units = 'ms';\n }\n }\n\n return value.toFixed(3) + units;\n}\n"]} |