UNPKG

12.5 kBPlain TextView Raw
1/*
2 * Copyright © 2019 Atomist, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17import {
18 DefaultExcludes,
19 GitProject,
20 Project,
21 projectUtils,
22} from "@atomist/automation-client";
23import {
24 AnyPush,
25 ExecuteGoalResult,
26 GoalInvocation,
27 GoalProjectListenerEvent,
28 GoalProjectListenerRegistration,
29 PushTest,
30} from "@atomist/sdm";
31import * as _ from "lodash";
32import { toArray } from "../../util/misc/array";
33import { CompressingGoalCache } from "./CompressingGoalCache";
34
35export const CacheInputGoalDataKey = "@atomist/sdm/input";
36export const CacheOutputGoalDataKey = "@atomist/sdm/output";
37
38/**
39 * Goal cache interface for storing and retrieving arbitrary files produced
40 * by the execution of a goal.
41 * @see FileSystemGoalCache`
42 */
43export interface GoalCache {
44
45 /**
46 * Add a set of files (or directories) to the cache.
47 * @param gi The goal invocation for which the cache needs to be stored.
48 * @param p The project where the files (or directories) reside.
49 * @param files The files (or directories) to be cached.
50 * @param classifier An optional classifier to identify the set of files (or directories to be cached).
51 */
52 put(gi: GoalInvocation, p: GitProject, files: string | string[], classifier?: string): Promise<void>;
53
54 /**
55 * Retrieve files from the cache.
56 * @param gi The goal invocation for which the cache needs to be restored.
57 * @param p he project where the files (or directories) need to be restored in.
58 * @param classifier Optionally the classifier of the cache for the files to be restored. If not defined,
59 * all caches for the GoalInvocation are restored.
60 */
61 retrieve(gi: GoalInvocation, p: GitProject, classifier?: string): Promise<void>;
62
63 /**
64 * Remove files from the cache.
65 * @param gi The goal invocation for which the cache needs to be removed.
66 * @param classifier Optionally the classifier of the cache for the files to be removed. If not defined,
67 * all classifiers are removed.
68 */
69 remove(gi: GoalInvocation, classifier?: string): Promise<void>;
70}
71
72/**
73 * Suitable for a limited set of files adhering to a pattern.
74 */
75export interface GlobFilePattern {
76 globPattern: string | string[];
77}
78
79/**
80 * Suitable for caching complete directories, possibly containing a lot of files.
81 */
82export interface DirectoryPattern {
83 directory: string;
84}
85
86export interface CacheEntry {
87 classifier: string;
88 pattern: GlobFilePattern | DirectoryPattern;
89}
90
91/**
92 * Core options for goal caching.
93 */
94export interface GoalCacheCoreOptions {
95 /**
96 * Optional push test on when to trigger caching
97 */
98 pushTest?: PushTest;
99 /**
100 * Optional listener functions that should be called when no cache entry is found.
101 */
102 onCacheMiss?: GoalProjectListenerRegistration | GoalProjectListenerRegistration[];
103}
104
105/**
106 * Options for putting goal cache entries.
107 */
108export interface GoalCacheOptions extends GoalCacheCoreOptions {
109 /**
110 * Collection of glob patterns with classifiers to determine which
111 * files need to be cached between goal invocations, possibly
112 * excluding paths using regular expressions.
113 */
114 entries: CacheEntry[];
115}
116
117/**
118 * Options for restoring goal cache entries.
119 */
120export interface GoalCacheRestoreOptions extends GoalCacheCoreOptions {
121 entries?: Array<{ classifier: string }>;
122}
123
124const DefaultGoalCache = new CompressingGoalCache();
125
126/**
127 * Goal listener that performs caching after a goal has been run.
128 * @param options The options for caching
129 * @param classifier Whether only a specific classifier, as defined in the options,
130 * needs to be cached. If omitted, all classifiers are cached.
131 * @param classifiers Additional classifiers that need to be created.
132 */
133export function cachePut(options: GoalCacheOptions,
134 classifier?: string,
135 ...classifiers: string[]): GoalProjectListenerRegistration {
136 const allClassifiers = [];
137 if (classifier) {
138 allClassifiers.push(classifier, ...(classifiers || []));
139 }
140
141 const entries = !!classifier ?
142 options.entries.filter(pattern => allClassifiers.includes(pattern.classifier)) :
143 options.entries;
144
145 const listenerName = `caching outputs`;
146
147 return {
148 name: listenerName,
149 listener: async (p: GitProject,
150 gi: GoalInvocation): Promise<void | ExecuteGoalResult> => {
151 if (!!isCacheEnabled(gi) && !process.env.ATOMIST_ISOLATED_GOAL_INIT) {
152 const goalCache = cacheStore(gi);
153 for (const entry of entries) {
154 const files = [];
155 if (isGlobFilePattern(entry.pattern)) {
156 files.push(...(await getFilePathsThroughPattern(p, entry.pattern.globPattern)));
157 } else if (isDirectoryPattern(entry.pattern)) {
158 files.push(entry.pattern.directory);
159 }
160 if (!_.isEmpty(files)) {
161 await goalCache.put(gi, p, files, entry.classifier);
162 }
163 }
164
165 // Set outputs on the goal data
166 const { goalEvent } = gi;
167 const data = JSON.parse(goalEvent.data || "{}");
168 const newData = {
169 [CacheOutputGoalDataKey]: [
170 ...(data[CacheOutputGoalDataKey] || []),
171 ...entries,
172 ],
173 };
174 goalEvent.data = JSON.stringify({
175 ...(JSON.parse(goalEvent.data || "{}")),
176 ...newData,
177 });
178 }
179 },
180 pushTest: options.pushTest,
181 events: [GoalProjectListenerEvent.after],
182 };
183}
184
185function isGlobFilePattern(toBeDetermined: any): toBeDetermined is GlobFilePattern {
186 return toBeDetermined.globPattern !== undefined;
187}
188
189function isDirectoryPattern(toBeDetermined: any): toBeDetermined is DirectoryPattern {
190 return toBeDetermined.directory !== undefined;
191}
192
193async function pushTestSucceeds(pushTest: PushTest, gi: GoalInvocation, p: GitProject): Promise<boolean> {
194 return (pushTest || AnyPush).mapping({
195 push: gi.goalEvent.push,
196 project: p,
197 id: gi.id,
198 configuration: gi.configuration,
199 addressChannels: gi.addressChannels,
200 context: gi.context,
201 preferences: gi.preferences,
202 credentials: gi.credentials,
203 });
204}
205
206async function invokeCacheMissListeners(optsToUse: GoalCacheOptions | GoalCacheRestoreOptions,
207 p: GitProject,
208 gi: GoalInvocation,
209 event: GoalProjectListenerEvent): Promise<void> {
210 for (const cacheMissFallback of toArray(optsToUse.onCacheMiss)) {
211 const allEvents = [GoalProjectListenerEvent.before, GoalProjectListenerEvent.after];
212 if ((cacheMissFallback.events || allEvents).filter(e => e === event).length > 0
213 && await pushTestSucceeds(cacheMissFallback.pushTest, gi, p)) {
214 await cacheMissFallback.listener(p, gi, event);
215 }
216 }
217}
218
219export const NoOpGoalProjectListenerRegistration: GoalProjectListenerRegistration = {
220 name: "NoOpListener",
221 listener: async () => {
222 },
223 pushTest: AnyPush,
224};
225
226/**
227 * Goal listener that performs cache restores before a goal has been run.
228 * @param options The options for caching
229 * @param classifier Whether only a specific classifier, as defined in the options,
230 * needs to be restored. If omitted, all classifiers defined in the options are restored.
231 * @param classifiers Additional classifiers that need to be restored.
232 */
233export function cacheRestore(options: GoalCacheRestoreOptions,
234 classifier?: string,
235 ...classifiers: string[]): GoalProjectListenerRegistration {
236 const allClassifiers = [];
237 if (classifier) {
238 allClassifiers.push(classifier, ...(classifiers || []));
239 }
240
241 const optsToUse: GoalCacheRestoreOptions = {
242 onCacheMiss: NoOpGoalProjectListenerRegistration,
243 ...options,
244 };
245
246 const classifiersToBeRestored = [];
247 if (allClassifiers.length > 0) {
248 classifiersToBeRestored.push(...allClassifiers);
249 } else {
250 classifiersToBeRestored.push(...optsToUse.entries.map(entry => entry.classifier));
251 }
252
253 const listenerName = `restoring inputs`;
254
255 return {
256 name: listenerName,
257 listener: async (p: GitProject,
258 gi: GoalInvocation,
259 event: GoalProjectListenerEvent): Promise<void | ExecuteGoalResult> => {
260 if (!!isCacheEnabled(gi)) {
261 const goalCache = cacheStore(gi);
262 for (const c of classifiersToBeRestored) {
263 try {
264 await goalCache.retrieve(gi, p, c);
265 } catch (e) {
266 await invokeCacheMissListeners(optsToUse, p, gi, event);
267 }
268 }
269 } else {
270 await invokeCacheMissListeners(optsToUse, p, gi, event);
271 }
272
273 // Set inputs on the goal data
274 const { goalEvent } = gi;
275 const data = JSON.parse(goalEvent.data || "{}");
276 const newData = {
277 [CacheInputGoalDataKey]: [
278 ...(data[CacheInputGoalDataKey] || []),
279 ...classifiersToBeRestored.map(c => ({
280 classifier: c,
281 })),
282 ],
283 };
284 goalEvent.data = JSON.stringify({
285 ...(JSON.parse(goalEvent.data || "{}")),
286 ...newData,
287 });
288 },
289 pushTest: optsToUse.pushTest,
290 events: [GoalProjectListenerEvent.before],
291 };
292}
293
294/**
295 * Goal listener that cleans up the cache restores after a goal has been run.
296 * @param options The options for caching
297 * @param classifier Whether only a specific classifier, as defined in the options,
298 * needs to be removed. If omitted, all classifiers are removed.
299 * @param classifiers Additional classifiers that need to be removed.
300 */
301export function cacheRemove(options: GoalCacheOptions,
302 classifier?: string,
303 ...classifiers: string[]): GoalProjectListenerRegistration {
304 const allClassifiers = [];
305 if (classifier) {
306 allClassifiers.push(...[classifier, ...classifiers]);
307 }
308
309 const classifiersToBeRemoved = [];
310 if (allClassifiers.length > 0) {
311 classifiersToBeRemoved.push(...allClassifiers);
312 } else {
313 classifiersToBeRemoved.push(...options.entries.map(entry => entry.classifier));
314 }
315
316 const listenerName = `removing outputs`;
317
318 return {
319 name: listenerName,
320 listener: async (p, gi) => {
321 if (!!isCacheEnabled(gi)) {
322 const goalCache = cacheStore(gi);
323
324 for (const c of classifiersToBeRemoved) {
325 await goalCache.remove(gi, c);
326 }
327 }
328 },
329 pushTest: options.pushTest,
330 events: [GoalProjectListenerEvent.after],
331 };
332}
333
334async function getFilePathsThroughPattern(project: Project, globPattern: string | string[]): Promise<string[]> {
335 const oldExcludes = DefaultExcludes;
336 DefaultExcludes.splice(0, DefaultExcludes.length); // necessary evil
337 try {
338 return await projectUtils.gatherFromFiles(project, globPattern, async f => f.path);
339 } finally {
340 DefaultExcludes.push(...oldExcludes);
341 }
342}
343
344function isCacheEnabled(gi: GoalInvocation): boolean {
345 return _.get(gi.configuration, "sdm.cache.enabled", false);
346}
347
348function cacheStore(gi: GoalInvocation): GoalCache {
349 return _.get(gi.configuration, "sdm.cache.store", DefaultGoalCache);
350}