1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 | const chalk = require('chalk');
|
14 | const { fetch } = require('@adobe/helix-fetch');
|
15 | const path = require('path');
|
16 | const _ = require('lodash/fp');
|
17 | const JunitPerformanceReport = require('./junit-utils');
|
18 | const AbstractCommand = require('./abstract.cmd.js');
|
19 |
|
20 | class PerformanceError extends Error {
|
21 |
|
22 | }
|
23 |
|
24 |
|
25 |
|
26 | class PerfCommand extends AbstractCommand {
|
27 | constructor(logger) {
|
28 | super(logger);
|
29 | this._location = 'London';
|
30 | this._device = 'MotorolaMotoG4';
|
31 | this._connection = 'regular3G';
|
32 | this._junit = null;
|
33 | this._fastly_namespace = null;
|
34 | this._fastly_auth = null;
|
35 | }
|
36 |
|
37 | withFastlyNamespace(value) {
|
38 | this._fastly_namespace = value;
|
39 | return this;
|
40 | }
|
41 |
|
42 | withFastlyAuth(value) {
|
43 | this._fastly_auth = value;
|
44 | return this;
|
45 | }
|
46 |
|
47 | withJunit(value) {
|
48 | if (value && value !== '') {
|
49 | this._junit = new JunitPerformanceReport().withOutfile(path.resolve(process.cwd(), value));
|
50 | }
|
51 | return this;
|
52 | }
|
53 |
|
54 | withLocation(value) {
|
55 | this._location = value;
|
56 | return this;
|
57 | }
|
58 |
|
59 | withDevice(value) {
|
60 | this._device = value;
|
61 | return this;
|
62 | }
|
63 |
|
64 | withConnection(value) {
|
65 | this._connection = value;
|
66 | return this;
|
67 | }
|
68 |
|
69 | getDefaultParams() {
|
70 | const defaultparams = {
|
71 | device: this._device,
|
72 | location: this._location,
|
73 | connection: this._connection,
|
74 | };
|
75 | return defaultparams;
|
76 | }
|
77 |
|
78 | getStrainParams(strain) {
|
79 | if (strain.perf) {
|
80 | return {
|
81 | device: strain.perf.device || this._device,
|
82 | location: strain.perf.location || this._location,
|
83 | connection: strain.perf.connection || this._connection,
|
84 | };
|
85 | }
|
86 | return this.getDefaultParams();
|
87 | }
|
88 |
|
89 | static formatScore(score, limit) {
|
90 | if (score >= limit) {
|
91 | return chalk.green.bold(score);
|
92 | }
|
93 | return chalk.red.bold(score) + chalk.red(' (failed)');
|
94 | }
|
95 |
|
96 | static formatMeasure(measure, limit) {
|
97 | if (measure <= limit) {
|
98 | return chalk.green.bold(measure);
|
99 | }
|
100 | return chalk.red.bold(measure) + chalk.red(' (failed)');
|
101 | }
|
102 |
|
103 | |
104 |
|
105 |
|
106 |
|
107 |
|
108 |
|
109 |
|
110 | format(metrics, name, limit) {
|
111 | const metric = metrics.filter((m) => m.name === name).length >= 1
|
112 | ? metrics.filter((m) => m.name === name)[0] : null;
|
113 | if (metric && metric.name.endsWith('-score')) {
|
114 | this.log.info(` ${chalk.gray(`${metric.label}: `)}${PerfCommand.formatScore(metric.value, limit)}`);
|
115 | return PerfCommand.formatScore(metric.value, limit).indexOf('(failed)') === -1;
|
116 | }
|
117 | if (metric) {
|
118 | this.log.info(` ${chalk.gray(`${metric.label}: `)}${PerfCommand.formatMeasure(metric.value, limit)}`);
|
119 | return PerfCommand.formatMeasure(metric.value, limit).indexOf('(failed)') === -1;
|
120 | }
|
121 | return undefined;
|
122 | }
|
123 |
|
124 | formatResponse(response, params = {}, strainname = 'default') {
|
125 | this.log.info(`\nResults for ${response.url} on ${response.device.title} (${response.connection.title}) from ${response.location.emoji} ${response.location.name} using ${strainname} strain.\n`);
|
126 | const strainresults = Object.keys(params).map((key) => {
|
127 | const value = params[key];
|
128 | if (Number.isInteger(value)) {
|
129 | return this.format(response.metrics, key, value);
|
130 | }
|
131 | return undefined;
|
132 | });
|
133 | if (strainresults.length === 0 || strainresults.every((val) => val === undefined)) {
|
134 | const perf = this.format(response.metrics, 'lighthouse-performance-score', 80);
|
135 | const access = this.format(response.metrics, 'lighthouse-accessibility-score', 80);
|
136 |
|
137 | return access && perf;
|
138 | }
|
139 |
|
140 | return strainresults.every((result) => result === true || result === undefined);
|
141 | }
|
142 |
|
143 | async run() {
|
144 | await this.init();
|
145 | this.log.info(chalk.green('Testing performance…'));
|
146 |
|
147 | const tests = this.config.strains
|
148 | .getByFilter(({ urls }) => urls.length)
|
149 | .map((strain) => {
|
150 | const { location, device, connection } = this.getStrainParams(strain);
|
151 | return strain.urls.map((url) => ({
|
152 | url,
|
153 | location,
|
154 | device,
|
155 | connection,
|
156 | strain: strain.name,
|
157 | ...strain.perf,
|
158 | }));
|
159 | });
|
160 | const flatttests = _.flatten(tests);
|
161 | const uri = 'https://adobeioruntime.net/api/v1/web/helix/helix-services/perf@v1';
|
162 |
|
163 | try {
|
164 | let response = await fetch(uri, {
|
165 | method: 'POST',
|
166 | json: {
|
167 | service: this._fastly_namespace,
|
168 | token: this._fastly_auth,
|
169 | tests: flatttests,
|
170 | },
|
171 | });
|
172 | if (!response.ok) {
|
173 | throw new Error(`${response.status} - "${await response.text()}"`);
|
174 | }
|
175 | const schedule = await response.json();
|
176 |
|
177 | let retries = 0;
|
178 | let results = [];
|
179 | while (retries < 10) {
|
180 | retries += 1;
|
181 | const completed = results.filter((res) => typeof res === 'object').length;
|
182 | console.log(chalk.yellow(`Waiting for test results (${completed}/${flatttests.length})`));
|
183 |
|
184 | response = await fetch(uri, {
|
185 | method: 'POST',
|
186 | json: {
|
187 | service: this._fastly_namespace,
|
188 | token: this._fastly_auth,
|
189 | tests: schedule,
|
190 | },
|
191 | });
|
192 | if (!response.ok) {
|
193 |
|
194 | throw new Error(`${response.status} - "${await response.text()}"`);
|
195 | }
|
196 |
|
197 | results = await response.json();
|
198 |
|
199 | if (results.reduce((p, uuid) => p && typeof uuid === 'object', true)) {
|
200 | break;
|
201 | }
|
202 | }
|
203 |
|
204 | let skipped = 0;
|
205 | const formatted = _.zip(results, flatttests).map(([res, test]) => {
|
206 | if (this._junit && typeof res === 'object') {
|
207 |
|
208 | this._junit.appendResults(res, test._thresholds, test.strain);
|
209 | }
|
210 | if (typeof res === 'object') {
|
211 |
|
212 | return this.formatResponse(res, test._thresholds, test.strain);
|
213 | }
|
214 | skipped += 1;
|
215 | console.log(chalk.yellow(`\nSkipped test for ${test.url} on ${test.strain}`));
|
216 | return undefined;
|
217 | });
|
218 |
|
219 | if (this._junit) {
|
220 | this._junit.writeResults();
|
221 | }
|
222 |
|
223 | const fail = formatted.filter((res) => res === false).length;
|
224 | const succeed = formatted.filter((res) => res === true).length;
|
225 | if (skipped) {
|
226 | console.log(chalk.yellow(`${skipped} tests skipped due to 10 minute timeout`));
|
227 | }
|
228 | if (fail && succeed) {
|
229 | this.log.error(chalk.yellow(`all tests completed with ${fail} failures and ${succeed} successes.`));
|
230 | throw new PerformanceError('Performance test failed partially');
|
231 | } else if (fail) {
|
232 | this.log.error(chalk.red(`all ${fail} tests failed.`));
|
233 | throw new PerformanceError('Performance test failed entirely');
|
234 | } else if (succeed) {
|
235 | this.log.info(chalk.green(`all ${succeed} tests succeeded.`));
|
236 | } else if (skipped) {
|
237 | throw new PerformanceError('Performance test skipped entirely');
|
238 | }
|
239 | } catch (e) {
|
240 | if (e instanceof PerformanceError) {
|
241 | throw e;
|
242 | }
|
243 | this.log.error(`Unable to run performance test ${e}`);
|
244 | throw new PerformanceError(`Unable to run performance test ${e}`);
|
245 | }
|
246 | }
|
247 | }
|
248 |
|
249 | module.exports = PerfCommand;
|