1 | import {RallyBase, lib, AbortError, Collection} from "./rally-tools.js";
|
2 | import {basename, resolve as pathResolve, dirname} from "path";
|
3 | import {cached, defineAssoc, spawn} from "./decorators.js";
|
4 | import {configObject} from "./config.js";
|
5 | import {loadLocals} from "./config-create";
|
6 | import Provider from "./providers.js";
|
7 | import Asset from "./asset.js";
|
8 |
|
9 |
|
10 | import {writeFileSync, readFileSync, pathTransform} from "./fswrap.js";
|
11 | import path from "path";
|
12 | import moment from "moment";
|
13 |
|
14 | let exists = {};
|
15 |
|
16 | class Preset extends RallyBase{
|
17 | constructor({path, remote, data, subProject} = {}){
|
18 |
|
19 | if(path){
|
20 | path = pathResolve(path);
|
21 | if(dirname(path).includes("silo-metadata")){
|
22 | throw new AbortError("Constructing preset from metadata file")
|
23 | }
|
24 | }
|
25 |
|
26 | super();
|
27 |
|
28 |
|
29 |
|
30 | if(path){
|
31 | if(exists[pathTransform(path)]) return exists[pathTransform(path)];
|
32 | exists[pathTransform(path)] = this;
|
33 | }
|
34 |
|
35 | this.meta = {};
|
36 | this.subproject = subProject;
|
37 | this.remote = remote
|
38 | if(lib.isLocalEnv(this.remote)){
|
39 | if(path){
|
40 | this.path = path;
|
41 | let pathspl = this.path.split(".");
|
42 | this.ext = pathspl[pathspl.length-1];
|
43 | try{
|
44 | this.code = this.getLocalCode();
|
45 | }catch(e){
|
46 | if(e.code === "ENOENT" && configObject.ignoreMissing){
|
47 | this.missing = true;
|
48 | return undefined;
|
49 | }else{
|
50 | log(chalk`{red Node Error} ${e.message}`);
|
51 | throw new AbortError("Could not load code of local file");
|
52 | }
|
53 | }
|
54 | let name = this.parseFilenameForName() || this.parseCodeForName();
|
55 | try{
|
56 | this.data = this.getLocalMetadata();
|
57 | this.isGeneric = true;
|
58 | name = this.name;
|
59 | }catch(e){
|
60 | log(chalk`{yellow Warning}: ${path} does not have a readable metadata file! Looking for ${this.localmetadatapath}`);
|
61 | this.data = Preset.newShell(name);
|
62 | this.isGeneric = false;
|
63 | }
|
64 | this.name = name;
|
65 | }else{
|
66 | this.data = Preset.newShell();
|
67 | }
|
68 | }else{
|
69 | this.data = data;
|
70 |
|
71 |
|
72 | this.isGeneric = false;
|
73 | }
|
74 | this.data.attributes.rallyConfiguration = undefined;
|
75 | this.data.attributes.systemManaged = undefined;
|
76 | }
|
77 |
|
78 | static async fromMetadata(path, subproject){
|
79 | let data;
|
80 | try{
|
81 | data = JSON.parse(readFileSync(path));
|
82 | }catch(e){
|
83 | if(e.code === "ENOENT" && configObject.ignoreMissing){
|
84 | return null;
|
85 | }else{
|
86 | throw e;
|
87 | }
|
88 | }
|
89 | let providerType = data.relationships.providerType.data.name;
|
90 | let provider = await Provider.getByName("DEV", providerType);
|
91 |
|
92 | if(!provider){
|
93 | log(chalk`{red The provider type {green ${providerType}} does not exist}`);
|
94 | log(chalk`{red Skipping {green ${path}}.}`);
|
95 | return null;
|
96 | }
|
97 |
|
98 | let ext = await provider.getFileExtension();
|
99 | let name = data.attributes.name;
|
100 |
|
101 | let realpath = Preset.getLocalPath(name, ext, subproject);
|
102 | return new Preset({path: realpath, subProject: subproject});
|
103 | }
|
104 |
|
105 | static newShell(name = undefined){
|
106 | return {
|
107 | "attributes": {
|
108 | "providerSettings": {
|
109 | "PresetName": name
|
110 | }
|
111 | },
|
112 | "relationships": {},
|
113 | "type": "presets",
|
114 | };
|
115 | }
|
116 | cleanup(){
|
117 | super.cleanup();
|
118 | delete this.attributes["createdAt"];
|
119 | delete this.attributes["updatedAt"];
|
120 | }
|
121 | async acclimatize(env){
|
122 | if(!this.isGeneric) throw new AbortError("Cannot acclimatize non-generics or shells");
|
123 | let providers = await Provider.getAll(env);
|
124 | let ptype = this.relationships["providerType"];
|
125 | ptype = ptype.data;
|
126 |
|
127 | let provider = providers.findByName(ptype.name);
|
128 | ptype.id = provider.id;
|
129 | }
|
130 | get test(){
|
131 | if(!this.code) return [];
|
132 |
|
133 | const regex = /[^-]autotest:\s?([\w\d_\-. \/]+)[\r\s\n]*?/gm;
|
134 | let match
|
135 | let matches = []
|
136 | while(match = regex.exec(this.code)){
|
137 | matches.push(match[1]);
|
138 | }
|
139 | return matches
|
140 | }
|
141 | async runTest(env){
|
142 | let remote = await Preset.getByName(env, this.name);
|
143 | for(let test of this.test){
|
144 | log("Tests...");
|
145 | let asset;
|
146 |
|
147 | if(test.startsWith("id")){
|
148 | let match = /id:\s*(\d+)/g.exec(test);
|
149 | if(!match){
|
150 | log(chalk`{red Could not parse autotest} ${test}.`);
|
151 | throw new AbortError("Could not properly parse the preset header");
|
152 | }
|
153 | asset = await Asset.getById(env, match[1]);
|
154 | }else{
|
155 | asset = await Asset.getByName(env, test);
|
156 | }
|
157 |
|
158 | if(!asset){
|
159 | log(chalk`{yellow No movie found}, skipping test.`);
|
160 | continue;
|
161 | }
|
162 |
|
163 | log(chalk`Starting job {green ${this.name}} on ${asset.chalkPrint(false)}... `);
|
164 | await asset.startEvaluate(remote.id);
|
165 | }
|
166 | }
|
167 | async resolve(){
|
168 | if(this.isGeneric) return;
|
169 |
|
170 | let proType = await this.resolveField(Provider, "providerType");
|
171 |
|
172 | this.ext = await proType.getFileExtension();
|
173 |
|
174 | this.isGeneric = true;
|
175 |
|
176 | return {proType};
|
177 | }
|
178 | async saveLocal(){
|
179 | await this.saveLocalMetadata();
|
180 | await this.saveLocalFile();
|
181 | }
|
182 | async saveLocalMetadata(){
|
183 | if(!this.isGeneric){
|
184 | await this.resolve();
|
185 | this.cleanup();
|
186 | }
|
187 | writeFileSync(this.localmetadatapath, JSON.stringify(this.data, null, 4));
|
188 | }
|
189 | async saveLocalFile(){
|
190 | writeFileSync(this.localpath, this.code);
|
191 | }
|
192 | async uploadRemote(env){
|
193 | await this.uploadCodeToEnv(env, true);
|
194 | }
|
195 | async save(env){
|
196 | this.saved = true;
|
197 | if(!this.isGeneric){
|
198 | await this.resolve();
|
199 | }
|
200 |
|
201 | this.cleanup();
|
202 | if(lib.isLocalEnv(env)){
|
203 | log(chalk`Saving preset {green ${this.name}} to {blue ${lib.envName(env)}}.`)
|
204 | await this.saveLocal();
|
205 | }else{
|
206 | await this.uploadRemote(env);
|
207 | }
|
208 | }
|
209 |
|
210 | async downloadCode(){
|
211 | if(!this.remote || this.code) return this.code;
|
212 | let code = await lib.makeAPIRequest({
|
213 | env: this.remote,
|
214 | path_full: this.data.links.providerData,
|
215 | json: false,
|
216 | });
|
217 |
|
218 |
|
219 |
|
220 |
|
221 |
|
222 |
|
223 | let headerRegex = /(^# .+[\r\n]+)+#+[\r\n]+/gim;
|
224 | let hasHeader = headerRegex.exec(code);
|
225 |
|
226 | if(hasHeader){
|
227 | this.header = code.substring(0, hasHeader[0].length - 1);
|
228 | code = code.substring(hasHeader[0].length);
|
229 | }
|
230 |
|
231 | return this.code = code;
|
232 | }
|
233 |
|
234 | get code(){
|
235 | if(this._code) return this._code;
|
236 | }
|
237 | set code(v){this._code = v;}
|
238 |
|
239 | chalkPrint(pad=true){
|
240 | let id = String("P-" + (this.remote && this.remote + "-" + this.id || "LOCAL"))
|
241 | let sub = "";
|
242 | if(this.subproject){
|
243 | sub = chalk`{yellow ${this.subproject}}`;
|
244 | }
|
245 | if(pad) id = id.padStart(10);
|
246 | if(this.name == undefined){
|
247 | return chalk`{green ${id}}: ${sub}{red ${this.path}}`;
|
248 | }else if(this.meta.proType){
|
249 | return chalk`{green ${id}}: ${sub}{red ${this.meta.proType.name}} {blue ${this.name}}`;
|
250 | }else{
|
251 | return chalk`{green ${id}}: ${sub}{blue ${this.name}}`;
|
252 | }
|
253 | }
|
254 | parseFilenameForName(){
|
255 | if(this.path.endsWith(".jinja") || this.path.endsWith(".json")){
|
256 | return basename(this.path)
|
257 | .replace("_", " ")
|
258 | .replace("-", " ")
|
259 | .replace(".json", "")
|
260 | .replace(".jinja", "");
|
261 | }
|
262 | }
|
263 |
|
264 | parseCodeForName(){
|
265 | const name_regex = /name\s?:\s([\w\d. \/]+)[\r\s\n]*?/;
|
266 | const match = name_regex.exec(this.code);
|
267 | if(match) return match[1];
|
268 | }
|
269 |
|
270 | findStringsInCode(strings){
|
271 | if(!this.code) return [];
|
272 |
|
273 | return strings.filter(str => {
|
274 | let regex = new RegExp(str);
|
275 | return !!this.code.match(regex);
|
276 | });
|
277 | }
|
278 | static getLocalPath(name, ext, subproject){
|
279 | return path.join(configObject.repodir, subproject || "", "silo-presets", name + "." + ext);
|
280 | }
|
281 | get localpath(){return Preset.getLocalPath(this.name, this.ext, this.subproject)}
|
282 |
|
283 | get path(){
|
284 | if(this._path) return this._path;
|
285 | }
|
286 | set path(val){
|
287 | this._path = val;
|
288 | }
|
289 | get name(){
|
290 | return this._nameOuter;
|
291 | }
|
292 | set name(val){
|
293 | if(!this._nameInner) this._nameInner = val;
|
294 | this._nameOuter = val;
|
295 | }
|
296 | set providerType(value){
|
297 | this.relationships["providerType"] = {
|
298 | data: {
|
299 | ...value,
|
300 | type: "providerTypes",
|
301 | }
|
302 | };
|
303 | }
|
304 | get localmetadatapath(){
|
305 | if(this.path){
|
306 | return this.path.replace("silo-presets", "silo-metadata").replace(new RegExp(this.ext + "$"), "json")
|
307 | }
|
308 | return path.join(configObject.repodir, this.subproject || "", "silo-metadata", this.name + ".json");
|
309 | }
|
310 | get immutable(){
|
311 | return this.name.includes("Constant") && !configObject.updateImmutable;
|
312 | }
|
313 | async uploadPresetData(env, id){
|
314 | if(this.code.trim() === "NOUPLOAD"){
|
315 | write(chalk`code skipped {yellow :)}, `);
|
316 | return;
|
317 | }
|
318 |
|
319 | let code = this.code;
|
320 | let headers = {};
|
321 |
|
322 | let providerName = this.relationships?.providerType?.data?.name;
|
323 | if(!configObject.skipHeader && (providerName === "SdviEvaluate" || providerName === "SdviEvalPro")){
|
324 | write(chalk`generate header, `);
|
325 | let repodir = configObject.repodir;
|
326 | let localpath = this.path.replace(repodir, "");
|
327 | if(localpath.startsWith("/")) localpath = localpath.substring(1);
|
328 |
|
329 | try{
|
330 | let {stdout: headerText} = await spawn(
|
331 | {noecho: true},
|
332 | "sh",
|
333 | [
|
334 | path.join(configObject.repodir, `bin/header.sh`),
|
335 | moment(Date.now()).format("ddd YYYY/MM/DD hh:mm:ssa"),
|
336 | localpath,
|
337 | ]
|
338 | );
|
339 | code = headerText + code;
|
340 | write(chalk`header ok, `);
|
341 | }catch(e){
|
342 | write(chalk`missing unix, `);
|
343 | }
|
344 | }
|
345 |
|
346 |
|
347 | if(providerName == "Vantage"){
|
348 | code = code.toString("base64");
|
349 | headers["Content-Transfer-Encoding"] = "base64";
|
350 | }
|
351 |
|
352 | let res = await lib.makeAPIRequest({
|
353 | env, path: `/presets/${id}/providerData`,
|
354 | body: code, method: "PUT", fullResponse: true, timeout: 10000,
|
355 | headers,
|
356 | });
|
357 | write(chalk`code up {yellow ${res.statusCode}}, `);
|
358 | }
|
359 | async grabMetadata(env){
|
360 | let remote = await Preset.getByName(env, this.name);
|
361 | this.isGeneric = false;
|
362 | if(!remote){
|
363 | throw new AbortError(`No file found on remote ${env} with name ${this.name}`);
|
364 | }
|
365 | this.data = remote.data;
|
366 | this.remote = env;
|
367 | }
|
368 |
|
369 | async deleteRemoteVersion(env, id=null){
|
370 | if(lib.isLocalEnv(env)) return false;
|
371 | if(!id){
|
372 | let remote = await Preset.getByName(env, this.name);
|
373 | id = remote.id;
|
374 | }
|
375 |
|
376 | return await lib.makeAPIRequest({
|
377 | env, path: `/presets/${id}`,
|
378 | method: "DELETE",
|
379 | });
|
380 | }
|
381 |
|
382 | async delete(){
|
383 | if(lib.isLocalEnv(this.remote)) return false;
|
384 |
|
385 | return await this.deleteRemoteVersion(this.remote, this.id);
|
386 | }
|
387 |
|
388 | async uploadCodeToEnv(env, includeMetadata, shouldTest = true){
|
389 | if(!this.name){
|
390 | let match;
|
391 | if(match = /^(#|["']{3})\s*EPH (\d+)/.exec(this.code.trim())){
|
392 | let a = await Asset.getById(env, Number(match[2]))
|
393 | return a.startEphemeralEvaluateIdeal(this);
|
394 | }else{
|
395 | log(chalk`Failed uploading {red ${this.path}}. No name found.`);
|
396 | return;
|
397 | }
|
398 | }
|
399 |
|
400 | write(chalk`Uploading preset {green ${this.name}} to {green ${env}}: `);
|
401 |
|
402 | if(this.immutable){
|
403 | log(chalk`{magenta IMMUTABLE}. Nothing to do.`);
|
404 | return;
|
405 | }
|
406 |
|
407 |
|
408 | let remote = await Preset.getByName(env, this.name);
|
409 |
|
410 | if(remote){
|
411 |
|
412 | write("replace, ");
|
413 | if(includeMetadata){
|
414 | let payload = {data: {attributes: this.data.attributes, type: "presets"}};
|
415 | if(this.relationships.tagNames){
|
416 | payload.relationships = {tagNames: this.relationships.tagNames};
|
417 | }
|
418 | let res = await lib.makeAPIRequest({
|
419 | env, path: `/presets/${remote.id}`, method: "PATCH",
|
420 | payload,
|
421 | fullResponse: true,
|
422 | });
|
423 | write(chalk`metadata {yellow ${res.statusCode}}, `);
|
424 | if(res.statusCode == 500){
|
425 | log(chalk`skipping code upload, did not successfully upload metadata`)
|
426 | return;
|
427 | }
|
428 | }
|
429 |
|
430 | await this.uploadPresetData(env, remote.id);
|
431 | }else{
|
432 | write("create, ");
|
433 | let metadata = {data: this.data};
|
434 | if(!this.relationships["providerType"]){
|
435 | throw new AbortError("Cannot acclimatize shelled presets. (try creating it on the env first)");
|
436 | }
|
437 |
|
438 | await this.acclimatize(env);
|
439 | write("Posting to create preset... ");
|
440 | let res = await lib.makeAPIRequest({
|
441 | env, path: `/presets`, method: "POST",
|
442 | payload: metadata, timeout: 5000,
|
443 | });
|
444 | let id = res.data.id;
|
445 | write(chalk`Created id {green ${id}}... Uploading Code... `);
|
446 | await this.uploadPresetData(env, id);
|
447 | }
|
448 | if(this.test[0] && shouldTest){
|
449 | await this.runTest(env);
|
450 | }else{
|
451 | log("No tests. Done.");
|
452 | }
|
453 | }
|
454 |
|
455 | getLocalMetadata(){
|
456 | return JSON.parse(readFileSync(this.localmetadatapath, "utf-8"));
|
457 | }
|
458 | getLocalCode(){
|
459 |
|
460 | return readFileSync(this.path, "utf-8");
|
461 | }
|
462 |
|
463 | parseHeaderInfo(){
|
464 | if(!this.header) return null;
|
465 | let abs = {
|
466 | built: /Built On:(.+)/.exec(this.header)[1]?.trim(),
|
467 | author: /Author:(.+)/.exec(this.header)[1]?.trim(),
|
468 | build: /Build:(.+)/.exec(this.header)[1]?.trim(),
|
469 | version: /Version:(.+)/.exec(this.header)[1]?.trim(),
|
470 | branch: /Branch:(.+)/.exec(this.header)[1]?.trim(),
|
471 | commit: /Commit:(.+)/.exec(this.header)[1]?.trim(),
|
472 | local: /Local File:(.+)/.exec(this.header)[1]?.trim(),
|
473 | }
|
474 |
|
475 | let tryFormats = [
|
476 | [true, "ddd MMM DD HH:mm:ss YYYY"],
|
477 | [false, "ddd YYYY/MM/DD LTS"],
|
478 | ];
|
479 |
|
480 | for(let [isUTC, format] of tryFormats){
|
481 | let date;
|
482 | if(isUTC){
|
483 | date = moment.utc(abs.built, format)
|
484 | }else{
|
485 | date = moment(abs.built, format)
|
486 | }
|
487 |
|
488 | if(!date.isValid()) continue;
|
489 |
|
490 | abs.offset = date.fromNow();
|
491 |
|
492 | break;
|
493 | }
|
494 |
|
495 | return abs;
|
496 | }
|
497 |
|
498 | async printRemoteInfo(env){
|
499 | let remote = await Preset.getByName(env, this.name);
|
500 | await remote.downloadCode();
|
501 | let i = remote.parseHeaderInfo();
|
502 |
|
503 | if(i){
|
504 | log(chalk`
|
505 | ENV: {red ${env}}, updated {yellow ~${i.offset}}
|
506 | Built on {blue ${i.built}} by {green ${i.author}}
|
507 | From ${i.build || "(unknown)"} on ${i.branch} ({yellow ${i.commit}})
|
508 | `.replace(/^[ \t]+/gim, "").trim());
|
509 | }else{
|
510 | log(chalk`No header on {red ${env}}`);
|
511 | }
|
512 | }
|
513 |
|
514 | async getInfo(envs){
|
515 | await this.printDepends();
|
516 | for(let env of envs.split(",")){
|
517 | await this.printRemoteInfo(env);
|
518 | }
|
519 | }
|
520 |
|
521 | async printDepends(indent=0, locals=null, seen={}){
|
522 | let includeRegex = /@include "(.+)"/gim;
|
523 | let includes = this.code
|
524 | .split("\n")
|
525 | .map(x => includeRegex.exec(x))
|
526 | .filter(x => x)
|
527 | .map(x => x[1]);
|
528 |
|
529 | if(!locals){
|
530 | locals = new Collection(await loadLocals("silo-presets", Preset));
|
531 | }
|
532 |
|
533 | log(Array(indent + 1).join(" ") + "- " + this.name);
|
534 |
|
535 | for(let include of includes){
|
536 | if(seen[include]){
|
537 | log(Array(indent + 1).join(" ") + " - (seen) " + include);
|
538 | }else{
|
539 | seen[include] = true
|
540 | await locals.findByName(include).printDepends(indent + 2, locals, seen);
|
541 | }
|
542 | }
|
543 | }
|
544 | }
|
545 |
|
546 | defineAssoc(Preset, "_nameInner", "data.attributes.providerSettings.PresetName");
|
547 | defineAssoc(Preset, "_nameOuter", "data.attributes.name");
|
548 | defineAssoc(Preset, "id", "data.id");
|
549 | defineAssoc(Preset, "attributes", "data.attributes");
|
550 | defineAssoc(Preset, "relationships", "data.relationships");
|
551 | defineAssoc(Preset, "remote", "meta.remote");
|
552 | defineAssoc(Preset, "_code", "meta.code");
|
553 | defineAssoc(Preset, "_path", "meta.path");
|
554 | defineAssoc(Preset, "isGeneric", "meta.isGeneric");
|
555 | defineAssoc(Preset, "ext", "meta.ext");
|
556 | defineAssoc(Preset, "subproject", "meta.project");
|
557 | defineAssoc(Preset, "metastring", "meta.metastring");
|
558 | Preset.endpoint = "presets";
|
559 |
|
560 | export default Preset;
|