1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 | import {
|
18 | GitHubRepoRef,
|
19 | GitProject,
|
20 | HttpMethod,
|
21 | isTokenCredentials,
|
22 | } from "@atomist/automation-client";
|
23 | import { resolvePlaceholders } from "@atomist/automation-client/lib/configuration";
|
24 | import {
|
25 | allSatisfied,
|
26 | Cancel,
|
27 | Goal,
|
28 | GoalWithFulfillment,
|
29 | ImmaterialGoals,
|
30 | Locking,
|
31 | PushTest,
|
32 | Queue,
|
33 | RepoContext,
|
34 | SdmGoalEvent,
|
35 | SoftwareDeliveryMachine,
|
36 | StatefulPushListenerInvocation,
|
37 | } from "@atomist/sdm";
|
38 | import * as yaml from "js-yaml";
|
39 | import * as stringify from "json-stringify-safe";
|
40 | import * as _ from "lodash";
|
41 | import {
|
42 | cachePut,
|
43 | cacheRestore,
|
44 | } from "../../goal/cache/goalCaching";
|
45 | import { item } from "../../goal/common/item";
|
46 | import {
|
47 | container,
|
48 | Container,
|
49 | ContainerProgressReporter,
|
50 | ContainerRegistration,
|
51 | ContainerSpecCallback,
|
52 | GoalContainerSpec,
|
53 | } from "../../goal/container/container";
|
54 | import { execute } from "../../goal/container/execute";
|
55 | import { toArray } from "../../util/misc/array";
|
56 | import { DeliveryGoals } from "../configure";
|
57 | import {
|
58 | mapTests,
|
59 | PushTestMaker,
|
60 | } from "./mapPushTests";
|
61 | import { resolvePlaceholder } from "./resolvePlaceholder";
|
62 | import { camelCase } from "./util";
|
63 |
|
64 |
|
65 |
|
66 | export type GoalMaker<G extends Record<string, any> = {}> =
|
67 | (sdm: SoftwareDeliveryMachine, params: G) => Promise<Goal> | Goal;
|
68 |
|
69 | type MapGoal = (goals: any,
|
70 | sdm: SoftwareDeliveryMachine,
|
71 | additionalGoals: DeliveryGoals,
|
72 | goalMakers: Record<string, GoalMaker>,
|
73 | additionalTests: Record<string, PushTest>,
|
74 | extensionTests: Record<string, PushTestMaker>) => Promise<Goal | Goal[]>;
|
75 |
|
76 | const MapContainer: MapGoal = async (goals: any,
|
77 | sdm: SoftwareDeliveryMachine,
|
78 | additionalGoals: DeliveryGoals,
|
79 | goalMakers: Record<string, GoalMaker>,
|
80 | additionalTests: Record<string, PushTest>,
|
81 | extensionTests: Record<string, PushTestMaker>) => {
|
82 | if (!!goals.containers) {
|
83 |
|
84 | if (!goals.name) {
|
85 | throw new Error(`Property 'name' missing on container goal:\n${JSON.stringify(goals, undefined, 2)}`);
|
86 | }
|
87 |
|
88 | const containers = [];
|
89 | for (const gc of goals.containers) {
|
90 | containers.push({
|
91 | ...gc,
|
92 | name: gc.name.replace(/ /g, "-"),
|
93 | test: !!gc.test ? await mapTests(gc.test, additionalTests, extensionTests) : undefined,
|
94 | });
|
95 | }
|
96 | const g = container(
|
97 | goals.name,
|
98 | {
|
99 | callback: containerCallback(),
|
100 | containers,
|
101 | volumes: toArray(goals.volumes),
|
102 | progressReporter: ContainerProgressReporter,
|
103 | input: goals.input,
|
104 | output: goals.output,
|
105 | parameters: goals.parameters,
|
106 | fulfillment: goals.fulfillment,
|
107 | });
|
108 | return g;
|
109 | }
|
110 |
|
111 | return undefined;
|
112 | };
|
113 |
|
114 | const MapExecute: MapGoal = async goals => {
|
115 | if (!!goals.execute) {
|
116 |
|
117 | if (!goals.name) {
|
118 | throw new Error(`Property 'name' missing on execute goal:\n${JSON.stringify(goals, undefined, 2)}`);
|
119 | }
|
120 |
|
121 | const g = goals.execute;
|
122 | return execute(g.name, {
|
123 | cmd: g.command || g.cmd,
|
124 | args: toArray(g.args),
|
125 | secrets: g.secrets,
|
126 | });
|
127 | }
|
128 |
|
129 | return undefined;
|
130 | };
|
131 |
|
132 | const MapImmaterial: MapGoal = async goals => {
|
133 | if (goals.use === "immaterial") {
|
134 | return ImmaterialGoals.andLock().goals;
|
135 | }
|
136 | return undefined;
|
137 | };
|
138 |
|
139 | const MapLock: MapGoal = async goals => {
|
140 | if (goals.use === "lock") {
|
141 | return Locking;
|
142 | }
|
143 | return undefined;
|
144 | };
|
145 |
|
146 | const MapQueue: MapGoal = async goals => {
|
147 | if (goals.use === "queue") {
|
148 | return new Queue({
|
149 | fetch: goals.fetch,
|
150 | concurrent: goals.concurrent,
|
151 | });
|
152 | }
|
153 | return undefined;
|
154 | };
|
155 |
|
156 | const MapCancel: MapGoal = async goals => {
|
157 | if (goals.use === "cancel") {
|
158 | return new Cancel({ goals: [], goalNames: toArray(goals.goals) });
|
159 | }
|
160 | return undefined;
|
161 | };
|
162 |
|
163 | const MapAdditional: MapGoal = async (goals: any,
|
164 | sdm: SoftwareDeliveryMachine,
|
165 | additionalGoals: DeliveryGoals) => {
|
166 | if (!!additionalGoals[goals.use]) {
|
167 | return additionalGoals[goals.use];
|
168 | }
|
169 | return undefined;
|
170 | };
|
171 |
|
172 | const MapReferenced: MapGoal = async (goals: any,
|
173 | sdm: SoftwareDeliveryMachine,
|
174 | additionalGoals: DeliveryGoals,
|
175 | goalMakers: Record<string, GoalMaker>,
|
176 | additionalTests: Record<string, PushTest>,
|
177 | extensionTests: Record<string, PushTestMaker>) => {
|
178 | const use = goals.use;
|
179 | if (!!use && use.includes("/") && !use.startsWith("@")) {
|
180 | const parameters = goals.parameters || {};
|
181 | const referencedGoal = await mapReferencedGoal(sdm, use, parameters);
|
182 | if (!!referencedGoal) {
|
183 | return mapGoals(
|
184 | sdm,
|
185 | _.merge({}, referencedGoal, (goals || {})),
|
186 | additionalGoals,
|
187 | goalMakers,
|
188 | additionalTests,
|
189 | extensionTests);
|
190 | }
|
191 | }
|
192 |
|
193 | return undefined;
|
194 | };
|
195 |
|
196 | const MapGoalMakers: MapGoal = async (goals: any,
|
197 | sdm: SoftwareDeliveryMachine,
|
198 | additionalGoals: DeliveryGoals,
|
199 | goalMakers: Record<string, GoalMaker>) => {
|
200 |
|
201 | const use = goals.use;
|
202 | if (!!use && !!goalMakers[use]) {
|
203 | const goalMaker = goalMakers[use];
|
204 | try {
|
205 | return goalMaker(sdm, (goals.parameters || {})) as any;
|
206 | } catch (e) {
|
207 | e.message = `Failed to make goal using ${use}: ${e.message}`;
|
208 | throw e;
|
209 | }
|
210 | }
|
211 | return undefined;
|
212 | };
|
213 |
|
214 | const MapFulfillment: MapGoal = async (goals: any) => {
|
215 | const regexp = /([@a-zA-Z-_]*)\/([a-zA-Z-_]*)(?:\/([a-zA-Z-_]*))?@?([a-zA-Z-_0-9\.]*)/i;
|
216 | const use = goals.use;
|
217 |
|
218 | if (!!use) {
|
219 | const match = regexp.exec(use);
|
220 | if (!!match && use.startsWith("@")) {
|
221 | return item(
|
222 | match[3].replace(/_/g, " "),
|
223 | `${match[1]}/${match[2]}`,
|
224 | {
|
225 | uniqueName: goals.name || match[3],
|
226 | parameters: goals.parameters,
|
227 | input: goals.input,
|
228 | output: goals.output,
|
229 | secrets: goals.secrets,
|
230 | });
|
231 | }
|
232 | }
|
233 |
|
234 | return undefined;
|
235 | };
|
236 |
|
237 | const MapGoals = [
|
238 | MapContainer,
|
239 | MapExecute,
|
240 | MapImmaterial,
|
241 | MapLock,
|
242 | MapCancel,
|
243 | MapQueue,
|
244 | MapAdditional,
|
245 | MapGoalMakers,
|
246 | MapReferenced,
|
247 | MapFulfillment,
|
248 | ];
|
249 |
|
250 | export async function mapGoals(sdm: SoftwareDeliveryMachine,
|
251 | goals: any,
|
252 | additionalGoals: DeliveryGoals,
|
253 | goalMakers: Record<string, GoalMaker>,
|
254 | additionalTests: Record<string, PushTest>,
|
255 | extensionTests: Record<string, PushTestMaker>): Promise<Goal | Goal[]> {
|
256 | if (Array.isArray(goals)) {
|
257 | const newGoals: any[] = [];
|
258 | for (const g of toArray(goals)) {
|
259 | newGoals.push(await mapGoals(sdm, g, additionalGoals, goalMakers, additionalTests, extensionTests));
|
260 | }
|
261 | return newGoals;
|
262 | } else {
|
263 | let goal;
|
264 | for (const mapGoal of MapGoals) {
|
265 | goal = await mapGoal(goals, sdm, additionalGoals, goalMakers, additionalTests, extensionTests);
|
266 | if (!!goal) {
|
267 | if (!Array.isArray(goal)) {
|
268 | addDetails(goal, goals);
|
269 |
|
270 |
|
271 | if (!(goal instanceof Container)) {
|
272 | addCaching(goal, goals);
|
273 | }
|
274 | }
|
275 | return goal;
|
276 | }
|
277 | }
|
278 | }
|
279 |
|
280 | throw new Error(`Unable to construct goal from '${stringify(goals)}'`);
|
281 | }
|
282 |
|
283 | function addDetails(goal: Goal, goals: any): Goal {
|
284 | (goal as any).definition = _.cloneDeep(goal.definition);
|
285 | if (goals.approval !== undefined) {
|
286 | goal.definition.approvalRequired = goals.approval;
|
287 | }
|
288 | if (goals.preApproval !== undefined) {
|
289 | goal.definition.preApprovalRequired = goals.preApproval;
|
290 | }
|
291 | if (goals.retry !== undefined) {
|
292 | goal.definition.retryFeasible = goals.retry;
|
293 | }
|
294 | if (!!goals.descriptions) {
|
295 | const descriptions = goals.descriptions;
|
296 | goal.definition.canceledDescription = descriptions.canceled;
|
297 | goal.definition.completedDescription = descriptions.completed;
|
298 | goal.definition.failedDescription = descriptions.failed;
|
299 | goal.definition.plannedDescription = descriptions.planned;
|
300 | goal.definition.requestedDescription = descriptions.requested;
|
301 | goal.definition.stoppedDescription = descriptions.stopped;
|
302 | goal.definition.waitingForApprovalDescription = descriptions.waitingForApproval;
|
303 | goal.definition.waitingForPreApprovalDescription = descriptions.waitingForPreApproval;
|
304 | goal.definition.workingDescription = descriptions.inProcess;
|
305 | }
|
306 | return goal;
|
307 | }
|
308 |
|
309 | function addCaching(goal: GoalWithFulfillment, goals: any): GoalWithFulfillment {
|
310 | if (!!goals?.input) {
|
311 | goal.withProjectListener(cacheRestore({ entries: toArray(goals.input) }));
|
312 | }
|
313 | if (!!goals?.output) {
|
314 | goal.withProjectListener(cachePut({ entries: toArray(goals.output) }));
|
315 | }
|
316 | return goal;
|
317 | }
|
318 |
|
319 | function containerCallback(): ContainerSpecCallback {
|
320 | return async (r, p, g, e, ctx) => {
|
321 | const pli: StatefulPushListenerInvocation = {
|
322 | ...ctx,
|
323 | push: e.push,
|
324 | project: p,
|
325 | };
|
326 | const containersToRemove = [];
|
327 | for (const gc of r.containers) {
|
328 | let test;
|
329 | if (Array.isArray((gc as any).test)) {
|
330 | test = allSatisfied(...(gc as any).test);
|
331 | } else {
|
332 | test = (gc as any).test;
|
333 | }
|
334 | if (!!test && !(await test.mapping(pli))) {
|
335 | containersToRemove.push(gc);
|
336 | }
|
337 | }
|
338 | const registration: ContainerRegistration = {
|
339 | ...r,
|
340 | containers: r.containers.filter(c => !containersToRemove.includes(c)),
|
341 | };
|
342 | return resolvePlaceholderContainerSpecCallback(registration, p, g, e, ctx);
|
343 | };
|
344 | }
|
345 |
|
346 | async function mapReferencedGoal(sdm: SoftwareDeliveryMachine,
|
347 | goalRef: string,
|
348 | parameters: Record<string, any>): Promise<any> {
|
349 | const regexp = /([a-zA-Z-_]*)\/([a-zA-Z-_]*)(?:\/([a-zA-Z-_]*))?@?([a-zA-Z-_0-9\.]*)/i;
|
350 | const match = regexp.exec(goalRef);
|
351 | if (!match) {
|
352 | return undefined;
|
353 | }
|
354 |
|
355 | const owner = match[1];
|
356 | const repo = match[2];
|
357 | const goalName = match[3];
|
358 | const goalNames = !!goalName ? [goalName] : [repo, repo.replace(/-goal/, "")];
|
359 | const ref = match[4] || "master";
|
360 |
|
361 |
|
362 | let token = sdm.configuration?.sdm?.github?.token || sdm.configuration?.sdm?.goal?.yaml?.token;
|
363 | if (!token) {
|
364 | const workspaceId = _.get(sdm.configuration, "workspaceIds[0]");
|
365 | if (!!workspaceId) {
|
366 | try {
|
367 | const creds = await sdm.configuration.sdm.credentialsResolver.eventHandlerCredentials(
|
368 | { graphClient: sdm.configuration.graphql.client.factory.create(workspaceId, sdm.configuration) } as any, GitHubRepoRef.from({
|
369 | owner: undefined,
|
370 | repo,
|
371 | }));
|
372 | if (!!creds && isTokenCredentials(creds)) {
|
373 | token = creds.token;
|
374 | _.set(sdm.configuration, "sdm.goal.yaml.token", token);
|
375 | }
|
376 | } catch (e) {
|
377 |
|
378 | }
|
379 | }
|
380 | }
|
381 |
|
382 | const url = `https://api.github.com/repos/${owner}/${repo}/contents/goal.yaml?ref=${ref}`;
|
383 |
|
384 | try {
|
385 | const cacheKey = `configuration.sdm.goal.definition.cache[${url}]`;
|
386 | const cachedDocuments = _.get(sdm, cacheKey);
|
387 | let documents;
|
388 |
|
389 | if (!!cachedDocuments) {
|
390 | documents = cachedDocuments;
|
391 | } else {
|
392 | const client = sdm.configuration.http.client.factory.create(url);
|
393 | const response = await client.exchange<{ content: string }>(url, {
|
394 | method: HttpMethod.Get,
|
395 | headers: {
|
396 | ...(!!token ? { Authorization: `Bearer ${token}` } : {}),
|
397 | },
|
398 | retry: { retries: 0 },
|
399 | });
|
400 | const content = Buffer.from(response.body.content, "base64").toString();
|
401 | documents = yaml.safeLoadAll(content);
|
402 | _.set(sdm, cacheKey, documents);
|
403 | }
|
404 |
|
405 | for (const document of documents) {
|
406 | for (const key in document) {
|
407 | if (document.hasOwnProperty(key) && goalNames.includes(key)) {
|
408 | const pdg = document[key];
|
409 | await resolvePlaceholders(pdg,
|
410 | value => resolvePlaceholder(value, undefined, {} as any, parameters, false));
|
411 | return camelCase(pdg);
|
412 | }
|
413 | }
|
414 | }
|
415 | } catch (e) {
|
416 | throw new Error(`Referenced goal '${goalRef}' can not be created: ${e.message}`);
|
417 | }
|
418 | return undefined;
|
419 | }
|
420 |
|
421 | async function resolvePlaceholderContainerSpecCallback(r: ContainerRegistration,
|
422 | p: GitProject,
|
423 | g: Container,
|
424 | e: SdmGoalEvent,
|
425 | ctx: RepoContext): Promise<GoalContainerSpec> {
|
426 | await resolvePlaceholders(r as any, value => resolvePlaceholder(value, e, ctx, (r as any).parameters));
|
427 | return r;
|
428 | }
|