1 |
|
2 |
|
3 |
|
4 |
|
5 |
|
6 |
|
7 |
|
8 |
|
9 |
|
10 |
|
11 |
|
12 |
|
13 |
|
14 |
|
15 |
|
16 |
|
17 | import {
|
18 | DefaultExcludes,
|
19 | GitProject,
|
20 | Project,
|
21 | projectUtils,
|
22 | } from "@atomist/automation-client";
|
23 | import {
|
24 | AnyPush,
|
25 | ExecuteGoalResult,
|
26 | GoalInvocation,
|
27 | GoalProjectListenerEvent,
|
28 | GoalProjectListenerRegistration,
|
29 | PushTest,
|
30 | } from "@atomist/sdm";
|
31 | import * as _ from "lodash";
|
32 | import { toArray } from "../../util/misc/array";
|
33 | import { CompressingGoalCache } from "./CompressingGoalCache";
|
34 |
|
35 | export const CacheInputGoalDataKey = "@atomist/sdm/input";
|
36 | export const CacheOutputGoalDataKey = "@atomist/sdm/output";
|
37 |
|
38 |
|
39 |
|
40 |
|
41 |
|
42 |
|
43 | export interface GoalCache {
|
44 |
|
45 | |
46 |
|
47 |
|
48 |
|
49 |
|
50 |
|
51 |
|
52 | put(gi: GoalInvocation, p: GitProject, files: string | string[], classifier?: string): Promise<void>;
|
53 |
|
54 | |
55 |
|
56 |
|
57 |
|
58 |
|
59 |
|
60 |
|
61 | retrieve(gi: GoalInvocation, p: GitProject, classifier?: string): Promise<void>;
|
62 |
|
63 | |
64 |
|
65 |
|
66 |
|
67 |
|
68 |
|
69 | remove(gi: GoalInvocation, classifier?: string): Promise<void>;
|
70 | }
|
71 |
|
72 |
|
73 |
|
74 |
|
75 | export interface GlobFilePattern {
|
76 | globPattern: string | string[];
|
77 | }
|
78 |
|
79 |
|
80 |
|
81 |
|
82 | export interface DirectoryPattern {
|
83 | directory: string;
|
84 | }
|
85 |
|
86 | export interface CacheEntry {
|
87 | classifier: string;
|
88 | pattern: GlobFilePattern | DirectoryPattern;
|
89 | }
|
90 |
|
91 |
|
92 |
|
93 |
|
94 | export interface GoalCacheCoreOptions {
|
95 | |
96 |
|
97 |
|
98 | pushTest?: PushTest;
|
99 | |
100 |
|
101 |
|
102 | onCacheMiss?: GoalProjectListenerRegistration | GoalProjectListenerRegistration[];
|
103 | }
|
104 |
|
105 |
|
106 |
|
107 |
|
108 | export interface GoalCacheOptions extends GoalCacheCoreOptions {
|
109 | |
110 |
|
111 |
|
112 |
|
113 |
|
114 | entries: CacheEntry[];
|
115 | }
|
116 |
|
117 |
|
118 |
|
119 |
|
120 | export interface GoalCacheRestoreOptions extends GoalCacheCoreOptions {
|
121 | entries?: Array<{ classifier: string }>;
|
122 | }
|
123 |
|
124 | const DefaultGoalCache = new CompressingGoalCache();
|
125 |
|
126 |
|
127 |
|
128 |
|
129 |
|
130 |
|
131 |
|
132 |
|
133 | export function cachePut(options: GoalCacheOptions,
|
134 | classifier?: string,
|
135 | ...classifiers: string[]): GoalProjectListenerRegistration {
|
136 | const allClassifiers = [];
|
137 | if (classifier) {
|
138 | allClassifiers.push(classifier, ...(classifiers || []));
|
139 | }
|
140 |
|
141 | const entries = !!classifier ?
|
142 | options.entries.filter(pattern => allClassifiers.includes(pattern.classifier)) :
|
143 | options.entries;
|
144 |
|
145 | const listenerName = `caching outputs`;
|
146 |
|
147 | return {
|
148 | name: listenerName,
|
149 | listener: async (p: GitProject,
|
150 | gi: GoalInvocation): Promise<void | ExecuteGoalResult> => {
|
151 | if (!!isCacheEnabled(gi) && !process.env.ATOMIST_ISOLATED_GOAL_INIT) {
|
152 | const goalCache = cacheStore(gi);
|
153 | for (const entry of entries) {
|
154 | const files = [];
|
155 | if (isGlobFilePattern(entry.pattern)) {
|
156 | files.push(...(await getFilePathsThroughPattern(p, entry.pattern.globPattern)));
|
157 | } else if (isDirectoryPattern(entry.pattern)) {
|
158 | files.push(entry.pattern.directory);
|
159 | }
|
160 | if (!_.isEmpty(files)) {
|
161 | await goalCache.put(gi, p, files, entry.classifier);
|
162 | }
|
163 | }
|
164 |
|
165 |
|
166 | const { goalEvent } = gi;
|
167 | const data = JSON.parse(goalEvent.data || "{}");
|
168 | const newData = {
|
169 | [CacheOutputGoalDataKey]: [
|
170 | ...(data[CacheOutputGoalDataKey] || []),
|
171 | ...entries,
|
172 | ],
|
173 | };
|
174 | goalEvent.data = JSON.stringify({
|
175 | ...(JSON.parse(goalEvent.data || "{}")),
|
176 | ...newData,
|
177 | });
|
178 | }
|
179 | },
|
180 | pushTest: options.pushTest,
|
181 | events: [GoalProjectListenerEvent.after],
|
182 | };
|
183 | }
|
184 |
|
185 | function isGlobFilePattern(toBeDetermined: any): toBeDetermined is GlobFilePattern {
|
186 | return toBeDetermined.globPattern !== undefined;
|
187 | }
|
188 |
|
189 | function isDirectoryPattern(toBeDetermined: any): toBeDetermined is DirectoryPattern {
|
190 | return toBeDetermined.directory !== undefined;
|
191 | }
|
192 |
|
193 | async function pushTestSucceeds(pushTest: PushTest, gi: GoalInvocation, p: GitProject): Promise<boolean> {
|
194 | return (pushTest || AnyPush).mapping({
|
195 | push: gi.goalEvent.push,
|
196 | project: p,
|
197 | id: gi.id,
|
198 | configuration: gi.configuration,
|
199 | addressChannels: gi.addressChannels,
|
200 | context: gi.context,
|
201 | preferences: gi.preferences,
|
202 | credentials: gi.credentials,
|
203 | });
|
204 | }
|
205 |
|
206 | async function invokeCacheMissListeners(optsToUse: GoalCacheOptions | GoalCacheRestoreOptions,
|
207 | p: GitProject,
|
208 | gi: GoalInvocation,
|
209 | event: GoalProjectListenerEvent): Promise<void> {
|
210 | for (const cacheMissFallback of toArray(optsToUse.onCacheMiss)) {
|
211 | const allEvents = [GoalProjectListenerEvent.before, GoalProjectListenerEvent.after];
|
212 | if ((cacheMissFallback.events || allEvents).filter(e => e === event).length > 0
|
213 | && await pushTestSucceeds(cacheMissFallback.pushTest, gi, p)) {
|
214 | await cacheMissFallback.listener(p, gi, event);
|
215 | }
|
216 | }
|
217 | }
|
218 |
|
219 | export const NoOpGoalProjectListenerRegistration: GoalProjectListenerRegistration = {
|
220 | name: "NoOpListener",
|
221 | listener: async () => {
|
222 | },
|
223 | pushTest: AnyPush,
|
224 | };
|
225 |
|
226 |
|
227 |
|
228 |
|
229 |
|
230 |
|
231 |
|
232 |
|
233 | export function cacheRestore(options: GoalCacheRestoreOptions,
|
234 | classifier?: string,
|
235 | ...classifiers: string[]): GoalProjectListenerRegistration {
|
236 | const allClassifiers = [];
|
237 | if (classifier) {
|
238 | allClassifiers.push(classifier, ...(classifiers || []));
|
239 | }
|
240 |
|
241 | const optsToUse: GoalCacheRestoreOptions = {
|
242 | onCacheMiss: NoOpGoalProjectListenerRegistration,
|
243 | ...options,
|
244 | };
|
245 |
|
246 | const classifiersToBeRestored = [];
|
247 | if (allClassifiers.length > 0) {
|
248 | classifiersToBeRestored.push(...allClassifiers);
|
249 | } else {
|
250 | classifiersToBeRestored.push(...optsToUse.entries.map(entry => entry.classifier));
|
251 | }
|
252 |
|
253 | const listenerName = `restoring inputs`;
|
254 |
|
255 | return {
|
256 | name: listenerName,
|
257 | listener: async (p: GitProject,
|
258 | gi: GoalInvocation,
|
259 | event: GoalProjectListenerEvent): Promise<void | ExecuteGoalResult> => {
|
260 | if (!!isCacheEnabled(gi)) {
|
261 | const goalCache = cacheStore(gi);
|
262 | for (const c of classifiersToBeRestored) {
|
263 | try {
|
264 | await goalCache.retrieve(gi, p, c);
|
265 | } catch (e) {
|
266 | await invokeCacheMissListeners(optsToUse, p, gi, event);
|
267 | }
|
268 | }
|
269 | } else {
|
270 | await invokeCacheMissListeners(optsToUse, p, gi, event);
|
271 | }
|
272 |
|
273 |
|
274 | const { goalEvent } = gi;
|
275 | const data = JSON.parse(goalEvent.data || "{}");
|
276 | const newData = {
|
277 | [CacheInputGoalDataKey]: [
|
278 | ...(data[CacheInputGoalDataKey] || []),
|
279 | ...classifiersToBeRestored.map(c => ({
|
280 | classifier: c,
|
281 | })),
|
282 | ],
|
283 | };
|
284 | goalEvent.data = JSON.stringify({
|
285 | ...(JSON.parse(goalEvent.data || "{}")),
|
286 | ...newData,
|
287 | });
|
288 | },
|
289 | pushTest: optsToUse.pushTest,
|
290 | events: [GoalProjectListenerEvent.before],
|
291 | };
|
292 | }
|
293 |
|
294 |
|
295 |
|
296 |
|
297 |
|
298 |
|
299 |
|
300 |
|
301 | export function cacheRemove(options: GoalCacheOptions,
|
302 | classifier?: string,
|
303 | ...classifiers: string[]): GoalProjectListenerRegistration {
|
304 | const allClassifiers = [];
|
305 | if (classifier) {
|
306 | allClassifiers.push(...[classifier, ...classifiers]);
|
307 | }
|
308 |
|
309 | const classifiersToBeRemoved = [];
|
310 | if (allClassifiers.length > 0) {
|
311 | classifiersToBeRemoved.push(...allClassifiers);
|
312 | } else {
|
313 | classifiersToBeRemoved.push(...options.entries.map(entry => entry.classifier));
|
314 | }
|
315 |
|
316 | const listenerName = `removing outputs`;
|
317 |
|
318 | return {
|
319 | name: listenerName,
|
320 | listener: async (p, gi) => {
|
321 | if (!!isCacheEnabled(gi)) {
|
322 | const goalCache = cacheStore(gi);
|
323 |
|
324 | for (const c of classifiersToBeRemoved) {
|
325 | await goalCache.remove(gi, c);
|
326 | }
|
327 | }
|
328 | },
|
329 | pushTest: options.pushTest,
|
330 | events: [GoalProjectListenerEvent.after],
|
331 | };
|
332 | }
|
333 |
|
334 | async function getFilePathsThroughPattern(project: Project, globPattern: string | string[]): Promise<string[]> {
|
335 | const oldExcludes = DefaultExcludes;
|
336 | DefaultExcludes.splice(0, DefaultExcludes.length);
|
337 | try {
|
338 | return await projectUtils.gatherFromFiles(project, globPattern, async f => f.path);
|
339 | } finally {
|
340 | DefaultExcludes.push(...oldExcludes);
|
341 | }
|
342 | }
|
343 |
|
344 | function isCacheEnabled(gi: GoalInvocation): boolean {
|
345 | return _.get(gi.configuration, "sdm.cache.enabled", false);
|
346 | }
|
347 |
|
348 | function cacheStore(gi: GoalInvocation): GoalCache {
|
349 | return _.get(gi.configuration, "sdm.cache.store", DefaultGoalCache);
|
350 | }
|