1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 | const fetchAPI = require('@adobe/helix-fetch');
|
16 | const fs = require('fs-extra');
|
17 | const path = require('path');
|
18 | const fastly = require('@adobe/fastly-native-promises');
|
19 | const chalk = require('chalk');
|
20 | const ProgressBar = require('progress');
|
21 | const glob = require('glob-to-regexp');
|
22 | const { HelixConfig } = require('@adobe/helix-shared');
|
23 | const AbstractCommand = require('./abstract.cmd.js');
|
24 | const GitUtils = require('./git-utils.js');
|
25 | const cliversion = require('../package.json').version;
|
26 |
|
27 | const { fetch } = process.env.HELIX_FETCH_FORCE_HTTP1
|
28 | ? fetchAPI.context({ httpsProtocols: ['http1'] })
|
29 | : fetchAPI;
|
30 |
|
31 | class RemotePublishCommand extends AbstractCommand {
|
32 | constructor(logger) {
|
33 | super(logger);
|
34 | this._wsk_auth = null;
|
35 | this._wsk_namespace = null;
|
36 | this._wsk_host = null;
|
37 | this._fastly_namespace = null;
|
38 | this._debug_key = null;
|
39 | this._fastly_auth = null;
|
40 | this._dryRun = false;
|
41 | this._publishAPI = 'https://adobeioruntime.net/api/v1/web/helix/helix-services/publish@v7';
|
42 | this._githubToken = '';
|
43 | this._updateBotConfig = false;
|
44 | this._configPurgeAPI = 'https://app.project-helix.io/config/purge';
|
45 | this._vcl = null;
|
46 | this._dispatchVersion = null;
|
47 | this._purge = 'soft';
|
48 | this._algoliaAppID = null;
|
49 | this._algoliaAPIKey = null;
|
50 | this._epsagonAppName = null;
|
51 | this._epsagonToken = null;
|
52 | this._coralogixAppName = null;
|
53 | this._coralogixToken = null;
|
54 | }
|
55 |
|
56 | tick(ticks = 1, message, name) {
|
57 | if (name === true) {
|
58 |
|
59 | name = message;
|
60 | }
|
61 | this.progressBar().tick(ticks, {
|
62 | action: name || '',
|
63 | });
|
64 | if (message) {
|
65 | this.log.log({
|
66 | progress: true,
|
67 | level: 'info',
|
68 | message,
|
69 | });
|
70 | }
|
71 | }
|
72 |
|
73 | progressBar() {
|
74 | if (!this._bar) {
|
75 | this._bar = new ProgressBar('Publishing [:bar] :action :etas', {
|
76 | total: 24 + (this._updateBotConfig ? 2 : 0),
|
77 | width: 50,
|
78 | renderThrottle: 1,
|
79 | stream: process.stdout,
|
80 | });
|
81 | }
|
82 | return this._bar;
|
83 | }
|
84 |
|
85 | withPublishAPI(value) {
|
86 | this._publishAPI = value;
|
87 | return this;
|
88 | }
|
89 |
|
90 | withWskHost(value) {
|
91 | this._wsk_host = value;
|
92 | return this;
|
93 | }
|
94 |
|
95 | withWskAuth(value) {
|
96 | this._wsk_auth = value;
|
97 | return this;
|
98 | }
|
99 |
|
100 | withWskNamespace(value) {
|
101 | this._wsk_namespace = value;
|
102 | return this;
|
103 | }
|
104 |
|
105 | withFastlyNamespace(value) {
|
106 | this._fastly_namespace = value;
|
107 | return this;
|
108 | }
|
109 |
|
110 | withFastlyAuth(value) {
|
111 | this._fastly_auth = value;
|
112 | return this;
|
113 | }
|
114 |
|
115 | withDryRun(value) {
|
116 | this._dryRun = value;
|
117 | return this;
|
118 | }
|
119 |
|
120 | withGithubToken(value) {
|
121 | this._githubToken = value;
|
122 | return this;
|
123 | }
|
124 |
|
125 | withUpdateBotConfig(value) {
|
126 | this._updateBotConfig = value;
|
127 | return this;
|
128 | }
|
129 |
|
130 | withConfigPurgeAPI(value) {
|
131 | this._configPurgeAPI = value;
|
132 | return this;
|
133 | }
|
134 |
|
135 | withPurge(value) {
|
136 | this._purge = value;
|
137 | return this;
|
138 | }
|
139 |
|
140 | withDebugKey(value) {
|
141 | this._debug_key = value;
|
142 | return this;
|
143 | }
|
144 |
|
145 | withAlgoliaAppID(value) {
|
146 | this._algoliaAppID = value;
|
147 | return this;
|
148 | }
|
149 |
|
150 | withAlgoliaAPIKey(value) {
|
151 | this._algoliaAPIKey = value;
|
152 | return this;
|
153 | }
|
154 |
|
155 | withEpsagonAppName(value) {
|
156 | this._epsagonAppName = value;
|
157 | return this;
|
158 | }
|
159 |
|
160 | withEpsagonToken(value) {
|
161 | this._epsagonToken = value;
|
162 | return this;
|
163 | }
|
164 |
|
165 | withCoralogixAppName(value) {
|
166 | this._coralogixAppName = value;
|
167 | return this;
|
168 | }
|
169 |
|
170 | withCoralogixToken(value) {
|
171 | this._coralogixToken = value;
|
172 | return this;
|
173 | }
|
174 |
|
175 | withFilter(only, exclude) {
|
176 | if (!(only || exclude)) {
|
177 | return this;
|
178 | }
|
179 | const globex = glob(only || exclude);
|
180 |
|
181 | const onlyfilter = (master, current) => {
|
182 | const includecurrent = current && current.name && globex.test(current.name);
|
183 | const includemaster = master && master.name && !includecurrent;
|
184 | if (includecurrent) {
|
185 | return current;
|
186 | }
|
187 | if (includemaster) {
|
188 | return master;
|
189 | }
|
190 | return undefined;
|
191 | };
|
192 |
|
193 | const excludefilter = (master, current) => {
|
194 | const includecurrent = current && current.name && !globex.test(current.name);
|
195 | const includemaster = master && master.name && !includecurrent;
|
196 | if (includecurrent) {
|
197 | return current;
|
198 | }
|
199 | if (includemaster) {
|
200 | return master;
|
201 | }
|
202 | return undefined;
|
203 | };
|
204 |
|
205 | if (only) {
|
206 | this._filter = onlyfilter;
|
207 | } else if (exclude) {
|
208 | this._filter = excludefilter;
|
209 | }
|
210 |
|
211 | return this;
|
212 | }
|
213 |
|
214 | |
215 |
|
216 |
|
217 |
|
218 |
|
219 |
|
220 | withCustomVCLs(vcls) {
|
221 | if (vcls && vcls.length > 0) {
|
222 | const vcl = {};
|
223 | vcls.forEach((file) => {
|
224 | try {
|
225 | const fullPath = path.resolve(this.directory, file);
|
226 | const name = path.basename(fullPath, '.vcl');
|
227 | const content = fs.readFileSync(fullPath).toString();
|
228 | vcl[name] = content;
|
229 | } catch (error) {
|
230 | this.log.error(`Cannot read provided custom vcl file ${file}`);
|
231 | throw error;
|
232 | }
|
233 | });
|
234 | this._vcl = vcl;
|
235 | }
|
236 | return this;
|
237 | }
|
238 |
|
239 | |
240 |
|
241 |
|
242 |
|
243 | withDispatchVersion(version) {
|
244 | this._dispatchVersion = version;
|
245 | return this;
|
246 | }
|
247 |
|
248 | showNextStep(dryrun) {
|
249 | this.progressBar().terminate();
|
250 | if (dryrun) {
|
251 | this.log.info(`✅ A new version has been prepared, but not activated. See version ${this._version} in the Fastly UI at:`);
|
252 | this.log.info(chalk.grey(`https://manage.fastly.com/configure/services/${this._fastly_namespace}/versions/${this._version}/domains`));
|
253 | } else {
|
254 | const urls = this.config.strains
|
255 | .getByFilter((strain) => strain.url)
|
256 | .map((strain) => strain.url);
|
257 |
|
258 | this.log.info(`✅ The following strains have been published and version ${this._version} is now online:`);
|
259 | this.config.strains.getByFilter((strain) => !!strain.url).forEach((strain) => {
|
260 | const { url } = strain;
|
261 | urls.push(url);
|
262 | this.log.info(`- ${strain.name}: ${url}`);
|
263 | });
|
264 |
|
265 | if (urls.length) {
|
266 | this.log.info('\nYou may now access your site using:');
|
267 | this.log.info(chalk.grey(`$ curl ${urls[0]}`));
|
268 | }
|
269 | }
|
270 | }
|
271 |
|
272 | serviceAddLogger() {
|
273 | return fetch('https://adobeioruntime.net/api/v1/web/helix/helix-services/logging@v1', {
|
274 | method: 'POST',
|
275 | json: {
|
276 | service: this._fastly_namespace,
|
277 | token: this._fastly_auth,
|
278 | version: this._version,
|
279 | coralogixkey: this._coralogixToken,
|
280 | coralogixapp: this._coralogixAppName,
|
281 | cliversion,
|
282 | },
|
283 | }).then(async (res) => {
|
284 | if (!res.ok) {
|
285 | const e = new Error(`${res.status} - "${await res.text()}"`);
|
286 | e.statusCode = e.status;
|
287 | throw e;
|
288 | }
|
289 | await res.buffer();
|
290 | this.tick(10, 'set up logging', true);
|
291 | }).catch((e) => {
|
292 | this.tick(10, 'failed to set up logging', true);
|
293 | this.log.warn(`Remote addlogger service failed ${e}`);
|
294 | });
|
295 | }
|
296 |
|
297 | purgeFastly() {
|
298 | if (this._dryRun || this._purge === 'skip') {
|
299 | this.tick(1, 'skipping cache purge');
|
300 | return false;
|
301 | }
|
302 |
|
303 | const ok = () => {
|
304 | this.tick(1, 'purged cache', true);
|
305 | return true;
|
306 | };
|
307 |
|
308 | const err = (e) => {
|
309 | this.tick(1, 'failed to purge cache', true);
|
310 | this.log.error(`Cache could not get purged ${e}`);
|
311 | throw new Error('Unable to purge cache: ');
|
312 | };
|
313 |
|
314 | if (this._purge === 'hard') {
|
315 | return this._fastly.purgeAll().then(ok).catch(err);
|
316 | }
|
317 | return this._fastly.softPurgeKey('all').then(ok).catch(err);
|
318 | }
|
319 |
|
320 | async servicePublish() {
|
321 | this.tick(1, 'preparing service config for Helix', true);
|
322 |
|
323 | if (this._filter) {
|
324 | this.log.debug('filtering');
|
325 | try {
|
326 | const content = await GitUtils.getRawContent('.', 'master', 'helix-config.yaml');
|
327 |
|
328 | const other = await new HelixConfig()
|
329 | .withSource(content.toString())
|
330 | .init();
|
331 |
|
332 | this.log.debug(`this: ${Array.from(this.config.strains.keys()).join(', ')}`);
|
333 | this.log.debug(`other: ${Array.from(other.strains.keys()).join(', ')}`);
|
334 | const merged = other.merge(this.config, this._filter);
|
335 | this.log.debug(Array.from(merged.strains.keys()).join(', '));
|
336 |
|
337 | this._helixConfig = merged;
|
338 | } catch (e) {
|
339 | this.log.error(`Cannot merge configuration from master. Do you have a helix-config.yaml commited in the master branch?
|
340 | ${e}`);
|
341 | throw new Error('Unable to merge configurations for selective publishing');
|
342 | }
|
343 | }
|
344 | const body = {
|
345 | indexconfig: this.indexConfig.toJSON(),
|
346 | algoliaappid: this._algoliaAppID,
|
347 | epsagontoken: this._epsagonToken,
|
348 | epsagonapp: this._epsagonAppName,
|
349 | configuration: this.config.toJSON(),
|
350 | service: this._fastly_namespace,
|
351 | token: this._fastly_auth,
|
352 | version: this._version,
|
353 | };
|
354 |
|
355 | if (this._vcl) {
|
356 | body.vcl = this._vcl;
|
357 | }
|
358 |
|
359 | if (this._dispatchVersion) {
|
360 | body.dispatchVersion = this._dispatchVersion;
|
361 | }
|
362 |
|
363 | return fetch(this._publishAPI, {
|
364 | method: 'POST',
|
365 | json: body,
|
366 | }).then(async (res) => {
|
367 | if (!res.ok) {
|
368 | const e = new Error(`${res.status} - "${await res.text()}"`);
|
369 | e.statusCode = res.status;
|
370 | throw e;
|
371 | }
|
372 | await res.buffer();
|
373 | this.tick(9, 'set service config up for Helix', true);
|
374 | return true;
|
375 | }).catch((e) => {
|
376 | this.tick(9, 'failed to set service config up for Helix', true);
|
377 | this.log.error(`Remote publish service failed ${e}`);
|
378 | throw new Error('Unable to setup service config');
|
379 | });
|
380 | }
|
381 |
|
382 | async updateFastlySecrets() {
|
383 | const jobs = [];
|
384 | if (this._wsk_auth) {
|
385 | const auth = this._fastly.writeDictItem(this._version, 'secrets', 'OPENWHISK_AUTH', this._wsk_auth);
|
386 | jobs.push(auth);
|
387 | }
|
388 | if (this._wsk_namespace) {
|
389 | const namespace = this._fastly.writeDictItem(this._version, 'secrets', 'OPENWHISK_NAMESPACE', this._wsk_namespace);
|
390 | jobs.push(namespace);
|
391 | }
|
392 | if (this._algoliaAPIKey) {
|
393 | const apikey = this._fastly.writeDictItem(this._version, 'secrets', 'ALGOLIA_API_KEY', this._algoliaAPIKey);
|
394 | jobs.push(apikey);
|
395 | }
|
396 | if (this._algoliaAppID) {
|
397 | const appid = this._fastly.writeDictItem(this._version, 'secrets', 'ALGOLIA_APP_ID', this._algoliaAppID);
|
398 | jobs.push(appid);
|
399 | }
|
400 | const token = this._fastly.writeDictItem(this._version, 'secrets', 'GITHUB_TOKEN', this._githubToken);
|
401 | jobs.push(token);
|
402 | const debugKey = this._fastly.writeDictItem(this._version, 'secrets', 'DEBUG_KEY', this._debug_key || this._fastly_namespace);
|
403 | jobs.push(debugKey);
|
404 | return Promise.all(jobs).then(() => {
|
405 | this.tick(2, 'enabled authentication', true);
|
406 | return true;
|
407 | }).catch((e) => {
|
408 | this.tick(2, 'failed to enable authentication', true);
|
409 | this.log.error(`failed to enable authentication ${e}`);
|
410 | throw new Error('Unable to set credentials');
|
411 | });
|
412 | }
|
413 |
|
414 | async updateBotConfig() {
|
415 | const repos = {};
|
416 | this.tick(1, 'updating helix-bot purge config', true);
|
417 | this._strainsToPublish.forEach((strain) => {
|
418 | const url = strain.content;
|
419 |
|
420 | const urlString = `${url.protocol}://${url.host}/${url.owner}/${url.repo}.git#${url.ref}`;
|
421 | if (!repos[urlString]) {
|
422 | repos[urlString] = {
|
423 | strains: [],
|
424 | key: `${url.owner}/${url.repo}#${url.ref}`,
|
425 | };
|
426 | }
|
427 | repos[urlString].strains.push(strain.name);
|
428 | });
|
429 |
|
430 | const response = await fetch(this._configPurgeAPI, {
|
431 | method: 'POST',
|
432 | json: {
|
433 | github_token: this._githubToken,
|
434 | content_repositories: Object.keys(repos),
|
435 | fastly_service_id: this._fastly_namespace,
|
436 | fastly_token: this._fastly_auth,
|
437 | },
|
438 | });
|
439 |
|
440 | if (!response.ok) {
|
441 | const e = new Error(`${response.status} - "${await response.text()}"`);
|
442 | e.statusCode = response.status;
|
443 | throw e;
|
444 | }
|
445 |
|
446 | this._botStatus = {
|
447 | repos,
|
448 | response: await response.json(),
|
449 | };
|
450 | this.tick(1, 'updated helix-bot purge config', true);
|
451 | }
|
452 |
|
453 | showHelixBotResponse() {
|
454 | if (!this._botStatus) {
|
455 | return;
|
456 | }
|
457 | const { repos, response } = this._botStatus;
|
458 |
|
459 | const reposNoBot = [];
|
460 | const reposUpdated = [];
|
461 | const reposErrors = [];
|
462 | Object.keys(repos).forEach((repoUrl) => {
|
463 | const repo = repos[repoUrl];
|
464 | const info = response[repo.key];
|
465 | if (!info) {
|
466 | this.log.error(`Internal error: ${repo.key} should be in the service response`);
|
467 | reposErrors.push(repo);
|
468 | return;
|
469 | }
|
470 | if (info.errors) {
|
471 | this.log.error(`${repo.key} update failed: ${info.errors}`);
|
472 | reposErrors.push(repo);
|
473 | return;
|
474 | }
|
475 | if (!info.installation_id) {
|
476 | reposNoBot.push(repo);
|
477 | return;
|
478 | }
|
479 | if (!info.config || !info.config.caches) {
|
480 | this.log.error(`Internal error: ${repo.key} status does not have configuration details.`);
|
481 | reposErrors.push(repo);
|
482 | return;
|
483 | }
|
484 |
|
485 | const cacheInfo = info.config.caches
|
486 | .find((cache) => cache.fastlyServiceId === this._fastly_namespace);
|
487 | if (!cacheInfo) {
|
488 | this.log.error(`Internal error: ${repo.key} status does have a configuration entry for given fastly service id.`);
|
489 | reposErrors.push(repo);
|
490 | return;
|
491 | }
|
492 | if (cacheInfo.errors) {
|
493 | this.log.error(`${repo.key} update failed for given fastly service id: ${cacheInfo.errors}`);
|
494 | reposErrors.push(repo);
|
495 | return;
|
496 | }
|
497 | reposUpdated.push(repo);
|
498 | });
|
499 |
|
500 | if (reposUpdated.length > 0) {
|
501 | this.log.info('');
|
502 | this.log.info('Updated the purge-configuration of the following repositories:');
|
503 | reposUpdated.forEach((repo) => {
|
504 | this.log.info(chalk`- {cyan ${repo.key}} {grey (${repo.strains.join(', ')})}`);
|
505 | });
|
506 | }
|
507 |
|
508 | if (reposErrors.length > 0) {
|
509 | this.log.info('');
|
510 | this.log.info('The purge-configuration of following repositories were not updated due to errors (see log for details):');
|
511 | reposErrors.forEach((repo) => {
|
512 | this.log.info(chalk`- {cyan ${repo.key}} {grey (${repo.strains.join(', ')})}`);
|
513 | });
|
514 | }
|
515 |
|
516 | if (reposNoBot.length > 0) {
|
517 | this.log.info('');
|
518 | this.log.info('The following repositories are referenced by strains but don\'t have the helix-bot setup:');
|
519 | reposNoBot.forEach((repo) => {
|
520 | this.log.info(chalk`- {cyan ${repo.key}} {grey (${repo.strains.join(', ')})}`);
|
521 | });
|
522 | this.log.info(chalk`\nVisit {blue https://github.com/apps/helix-bot} to manage the helix bot installations.`);
|
523 | }
|
524 | }
|
525 |
|
526 | async init() {
|
527 | await super.init();
|
528 | this._fastly = fastly(this._fastly_auth, this._fastly_namespace);
|
529 |
|
530 |
|
531 | this._strainsToPublish = this.config.strains.getByFilter((strain) => {
|
532 | if (strain.isProxy()) {
|
533 | this.log.debug(`ignoring proxy strain ${strain.name}`);
|
534 | return false;
|
535 | }
|
536 |
|
537 | if (!strain.package) {
|
538 | this.log.debug(`ignoring unaffected strain ${strain.name}`);
|
539 | return false;
|
540 | }
|
541 | return true;
|
542 | });
|
543 | }
|
544 |
|
545 | async run() {
|
546 | await this.init();
|
547 | if (this._strainsToPublish.length === 0) {
|
548 | this.log.warn(chalk`None of the strains contains {cyan package} information. Aborting command.`);
|
549 | return;
|
550 | }
|
551 | try {
|
552 | this.tick(1, 'preparing fastly transaction', true);
|
553 | await this._fastly.transact(async (version) => {
|
554 | this._version = version;
|
555 | await this.servicePublish();
|
556 | await this.serviceAddLogger();
|
557 | await this.updateFastlySecrets();
|
558 | }, !this._dryRun);
|
559 | if (this._updateBotConfig) {
|
560 | await this.updateBotConfig();
|
561 | }
|
562 | await this.purgeFastly();
|
563 | this.showHelixBotResponse();
|
564 | this.showNextStep(this._dryRun);
|
565 | } catch (e) {
|
566 | const message = 'Error while running the Publish command';
|
567 | if (e.statusCode === 403) {
|
568 | throw new Error(`${message}:
|
569 | The provided GITHUB_TOKEN is not authorized to act on behalf
|
570 | of the Helix Bot and can therefore not be used to update the purge config.
|
571 | You can generate a new token by running 'hlx auth'`);
|
572 | } else {
|
573 | this.log.error(`${message}: ${e.stack}`, e);
|
574 | throw new Error(message, e);
|
575 | }
|
576 | }
|
577 | }
|
578 | }
|
579 | module.exports = RemotePublishCommand;
|