1 | var fileAndFolderUtils = require('../file-and-folder-utils');
|
2 | var SlJsInfra = require('sl-js-infra').SlJsInfra;
|
3 | var IgnoredFilesHandler = SlJsInfra.IgnoredFilesHandler;
|
4 | var glob = require("glob");
|
5 | var path = require('path');
|
6 | var fs = require('fs');
|
7 | var ActualFile = require('../file-system/actual-file');
|
8 | var utils = require('../utils');
|
9 | var consoleFileLogger = require("../sl-logger.js").consoleFileLogger;
|
10 | var FileSignature = require('./file-signature.js');
|
11 | var GeneratedFileSignature = require('./generated-file-signature');
|
12 | var micromatch = require('micromatch');
|
13 | var MarkupFilesParser = require('../markup-files-parser');
|
14 | var DefaultValuesResolver = require('../default-values-resolver');
|
15 | var EntitiesMapper = require("../entities-mapper");
|
16 | var SourceDataEnricher = require("./source-data-enricher");
|
17 | var globalErrorHandler = require('../global-error-handler');
|
18 | var globalMethodIndexer = require("./globalMethodIndexer");
|
19 |
|
20 |
|
21 | var BRANCHES_KEY = "branches";
|
22 | var BRANCH_ELEMENT_TYPE = "branch";
|
23 | var METHODS_KEY = "methods";
|
24 | var METHOD_ELEMENT_TYPE = "method";
|
25 | var METHOD_INDEX_FOR_GLOBAL_BRANCHES = -1;
|
26 |
|
27 |
|
28 | function FilesMapping(handlerParams, slMapper, logger) {
|
29 | this.handlerParams = handlerParams || {};
|
30 | this.options = this.handlerParams.buildArguments;
|
31 | this.finalOutput = this.handlerParams.finalOutput;
|
32 | this.matchedFiles = [];
|
33 | this.fileDataByFileName = {};
|
34 | this.fileSignatures = [];
|
35 | this.handlerParams.processedFiles = [];
|
36 | this.erroredFiles = [];
|
37 | this.finalFilesData = [];
|
38 | this.logger = logger;
|
39 | this.log = this._createLogger(logger, "FilesMapping");
|
40 | this.slIgnoreScan = this._createSlIgnore(IgnoredFilesHandler.SCAN_IGNORE_FILE_NAME);
|
41 | this.slIgnoreGenerated = this._createSlIgnore(IgnoredFilesHandler.GENERATED_IGNORE_FILE_NAME);
|
42 | this.projectRoot = this._resolveProjectRoot();
|
43 | this.shortNameToFileMapper = null;
|
44 | this.slMapper = slMapper;
|
45 | this.counters = {
|
46 | methods: 0,
|
47 | branches: 0
|
48 | };
|
49 | }
|
50 |
|
51 | FilesMapping.prototype.process = function () {
|
52 | var context = this
|
53 | var promise = new Promise(function (resolve) {
|
54 | context.log.info("Starting to scan the workspace.");
|
55 | context._scanWorkspaceForIncludedFilesAndPopulateFilesArray();
|
56 | context.matchedFiles.forEach(function (matchedFile, fileIdx) {
|
57 | var isLastFile = (fileIdx == context.matchedFiles.length - 1);
|
58 | context._fillFilesData(matchedFile, fileIdx, isLastFile);
|
59 | });
|
60 | context.fileDataByFileName = {};
|
61 | context.enrichBuildData();
|
62 | context.fileSignatures.forEach(function (fileSignature) {
|
63 | context._mapToServerEntities(fileSignature.createDTO());
|
64 | });
|
65 | if (context.fileSignatures.length > 0) {
|
66 | context.log.info("Processed the following files:");
|
67 | context.fileSignatures.forEach(function (file, idx) {
|
68 | context.log.info("\t[" + idx + "] Filename: '" + file.relativePath + "'");
|
69 | });
|
70 | }
|
71 | var isSuccess = true;
|
72 | if (context.erroredFiles.length) {
|
73 | var message = context._getErroredFilesMessage();
|
74 | context.log.warn(message);
|
75 | isSuccess = false;
|
76 | }
|
77 | if (SlJsInfra.SlEnvVars.isUseNewUniqueId()) {
|
78 | context._mapShortNamesToFiles();
|
79 | context._convertUniqueIds();
|
80 | }
|
81 | context.finalOutput.files = context.finalFilesData;
|
82 | context.finalOutput.counters = context.counters;
|
83 | return resolve(isSuccess);
|
84 | });
|
85 | return promise;
|
86 | }
|
87 |
|
88 | FilesMapping.prototype._mapToServerEntities = function(fileSignature){
|
89 | var context = this;
|
90 | fileSignature.methods.forEach(function(m){
|
91 | if(context.slMapper) {
|
92 | context.slMapper.mapMethod(m);
|
93 | }
|
94 | var methodFile = context._getCreateFileData(m.srcData.relativeFilename, m.srcData.absoluteFilename);
|
95 | methodFile.methods.push(EntitiesMapper.toServerMethod(m));
|
96 | context.counters.methods++;
|
97 | })
|
98 | fileSignature.branches.forEach(function(b){
|
99 | if(context.slMapper) {
|
100 | context.slMapper.mapBranch(b);
|
101 | }
|
102 |
|
103 | if(globalMethodIndexer.isMethodIgnored(b.enclosingMethodIdx)){
|
104 | b.enclosingMethodIdx = METHOD_INDEX_FOR_GLOBAL_BRANCHES;
|
105 | }
|
106 | var branchesFile = context._getCreateFileData(b.srcData.relativeFilename , b.srcData.absoluteFilename);
|
107 | branchesFile.branches.push(EntitiesMapper.toServerBranch(b));
|
108 | context.counters.branches++;
|
109 | })
|
110 | }
|
111 |
|
112 |
|
113 | FilesMapping.prototype._scanWorkspaceForIncludedFilesAndPopulateFilesArray = function () {
|
114 | var context = this
|
115 | context.options.includedFiles.forEach(function (inclusionPattern) {
|
116 | var matchedFiles = glob.sync(inclusionPattern,
|
117 | {
|
118 | cwd: context.options.workspacepath,
|
119 | ignore: context.options.excludedFiles
|
120 | });
|
121 |
|
122 | matchedFiles.forEach(function (matchedFile) {
|
123 | if (!context.fileDataByFileName[matchedFile]) {
|
124 | context.fileDataByFileName[matchedFile] = true;
|
125 | context.matchedFiles.push(matchedFile);
|
126 | }
|
127 | });
|
128 | });
|
129 | }
|
130 |
|
131 | FilesMapping.prototype._fillFilesData = function (matchedFile, fileIdx, isLastFile) {
|
132 | var absolutePath = path.resolve(this.options.workspacepath, matchedFile);
|
133 | absolutePath = utils.adjustPathSlashes(absolutePath);
|
134 | var shouldSkipCurrFile = this._shouldSkipFile(absolutePath, matchedFile, fileIdx);
|
135 | if (shouldSkipCurrFile)
|
136 | return;
|
137 | this.handlerParams.processedFiles.push(matchedFile);
|
138 | this.log.info('Processing [' + (fileIdx + 1) + '/' + this.matchedFiles.length + '] ' + absolutePath);
|
139 | var fileSig = this._generateFileSignature(absolutePath);
|
140 | if (!fileSig.hasError) {
|
141 | fileSig.methods = this._getIncludedMethods(fileSig);
|
142 | fileSig.branches = this._getIncludedBranches(fileSig);
|
143 | this.fileSignatures.push(fileSig);
|
144 | }
|
145 | else {
|
146 | this.erroredFiles.push(matchedFile);
|
147 | }
|
148 | }
|
149 |
|
150 | FilesMapping.prototype._shouldSkipFile = function (fullFilename, relativeFileName, fileIdx) {
|
151 | var filePath = fs.statSync(fullFilename);
|
152 | if (!filePath.isFile()) {
|
153 | this.log.info(relativeFileName + " is not a file.")
|
154 | return true;
|
155 | }
|
156 |
|
157 | if (this.slIgnoreScan.denies(relativeFileName)) {
|
158 | this.log.info('Excluded [' + (fileIdx + 1) + '/' + this.matchedFiles.length + '] ' + fullFilename);
|
159 |
|
160 | return true;
|
161 | }
|
162 | return false;
|
163 | }
|
164 |
|
165 | function createFileSignature(absolutePath, relativePath, fileContent, sourceMaps, opts, scmFilesContainer) {
|
166 | if(sourceMaps) {
|
167 | return GeneratedFileSignature.create(absolutePath, relativePath, fileContent, sourceMaps, opts, scmFilesContainer);
|
168 | }else {
|
169 | return FileSignature.create(absolutePath, relativePath, fileContent, sourceMaps, opts, scmFilesContainer);
|
170 | }
|
171 | }
|
172 |
|
173 | FilesMapping.prototype._generateFileSignature = function (absolutePath) {
|
174 | var relativePath = absolutePath.replace(this.projectRoot, "");
|
175 | if(relativePath.indexOf("/") === 0) {
|
176 | relativePath = relativePath.substring(1);
|
177 | }
|
178 | var isBranchCoverage = this.handlerParams.cfg.useBranchCoverage || this.options.usebranchcoverage;
|
179 | var fileObject = this._createActualFile(relativePath, absolutePath);
|
180 | var fileContent = fileObject.getContent();
|
181 | var sourceMaps = fileObject.readSourceMaps(this.logger);
|
182 |
|
183 | if (this._isMarkupFile(relativePath)) {
|
184 | fileContent = this._getJavaScriptFromMarkupFile(absolutePath, fileContent);
|
185 | }
|
186 | var opts = {
|
187 | es6Modules: true,
|
188 | babylonPlugins: ["typescript", "classProperties"].concat(this.options.babylonPlugins),
|
189 | projectRoot: this.options.projectRoot,
|
190 | scmRootDir: this.handlerParams.scmRootDir,
|
191 | isBranchCoverage: this.handlerParams.cfg.useBranchCoverage || this.options.usebranchcoverage
|
192 | };
|
193 | try {
|
194 | var fileSignature = createFileSignature(absolutePath, relativePath, fileContent, sourceMaps, opts, this.handlerParams.scmFilesContainer);
|
195 | return fileSignature;
|
196 | }catch (e) {
|
197 | this.logger.error("Failed to create signature for '%s'. Error '%s'" , absolutePath, e.message);
|
198 | this.logger.debug(e.stack);
|
199 | return {
|
200 | hasError: true
|
201 | };
|
202 | }
|
203 | };
|
204 |
|
205 | FilesMapping.prototype._createLogger = function (logger, className) {
|
206 | var log = logger ? logger.child({
|
207 | className: className
|
208 | }) : consoleFileLogger(className);
|
209 | return log;
|
210 | }
|
211 |
|
212 |
|
213 | FilesMapping.prototype._getIncludedMethods = function (fileSig) {
|
214 | if (!fileSig.methods)
|
215 | return [];
|
216 | return this._getIncludeElements(fileSig, METHODS_KEY, METHOD_ELEMENT_TYPE);;
|
217 | }
|
218 |
|
219 |
|
220 |
|
221 | FilesMapping.prototype._getIncludedBranches = function (fileSig) {
|
222 | if (!fileSig.branches)
|
223 | return []
|
224 | return this._getIncludeElements(fileSig, BRANCHES_KEY, BRANCH_ELEMENT_TYPE);
|
225 |
|
226 | }
|
227 |
|
228 | FilesMapping.prototype._getIncludeElements = function (fileSig, key, elementType) {
|
229 | var includedElements = [];
|
230 | var context = this;
|
231 | fileSig[key].forEach(function (element) {
|
232 | if (fileSig.hasSourceMaps) {
|
233 | context._handleGeneratedElement(element, includedElements, elementType);
|
234 | } else {
|
235 | includedElements.push(element);
|
236 | }
|
237 | });
|
238 | return includedElements;
|
239 | }
|
240 |
|
241 | FilesMapping.prototype._handleGeneratedElement = function (element, elementsArr, elementType) {
|
242 | if (this.slIgnoreGenerated.accepts(element.srcData.relativeFilename)) {
|
243 | elementsArr.push(element);
|
244 | } else {
|
245 | if(elementType === METHOD_ELEMENT_TYPE){
|
246 | globalMethodIndexer.setIgnoredMethod(element.idxInMapping);
|
247 | }
|
248 | this.log.info('Rejecting ' + elementType + ' from file: ' + element.srcData.relativeFilename);
|
249 | }
|
250 | }
|
251 |
|
252 | FilesMapping.prototype._getErroredFilesMessage = function () {
|
253 | return "The following files failed to be parsed and will not be sent to the server:\n" +
|
254 | this.erroredFiles.join('\n') +
|
255 | "\n\t(Hint: if the file use the es6 module syntax, pass the \"--es6Modules\" switch)";
|
256 | }
|
257 |
|
258 | FilesMapping.prototype._getCreateFileData = function (filename, absoluteFileName) {
|
259 | if (!this.fileDataByFileName[filename]) {
|
260 | var fileData = this._createNewFileData(filename, absoluteFileName);
|
261 | this.fileDataByFileName[filename] = fileData;
|
262 | this.finalFilesData.push(fileData);
|
263 | }
|
264 | return this.fileDataByFileName[filename];
|
265 | }
|
266 |
|
267 | FilesMapping.prototype._createNewFileData = function (filename, absoluteFileName) {
|
268 | var gitPath = this._resolvePhysicalPath(absoluteFileName) || filename;
|
269 | var newFile = {
|
270 | logicalPath: filename,
|
271 | physicalPath: gitPath,
|
272 | methods: [],
|
273 | lines: [],
|
274 | branches: []
|
275 | };
|
276 |
|
277 | if (this._shouldAddCommitsArray()) {
|
278 | newFile.commitIndexes = this.handlerParams.fileToCommitsMap[gitPath] || [];
|
279 | } else {
|
280 | this._logCommitsArrayNotFound();
|
281 | }
|
282 |
|
283 | return newFile;
|
284 | }
|
285 |
|
286 | FilesMapping.prototype._resolvePhysicalPath = function (absoluteFileName) {
|
287 | if (!this.handlerParams.scmRootDir) {
|
288 | this.logger.debug("scmRootDir not found");
|
289 | return;
|
290 | }
|
291 | absoluteFileName = utils.adjustPathSlashes(absoluteFileName);
|
292 | var scmRootDir = utils.adjustPathSlashes(this.handlerParams.scmRootDir);
|
293 | var physicalPath = absoluteFileName.replace(scmRootDir, "");
|
294 | if (physicalPath.indexOf("/") === 0) {
|
295 | physicalPath = physicalPath.substring(1);
|
296 | }
|
297 | return physicalPath;
|
298 | };
|
299 |
|
300 | FilesMapping.prototype._shouldAddCommitsArray = function () {
|
301 | return this.options && this.options.sendContributors && this.handlerParams.fileToCommitsMap;
|
302 | }
|
303 |
|
304 | FilesMapping.prototype._logCommitsArrayNotFound = function () {
|
305 | var fieldsMap = {
|
306 | opts: !this.options,
|
307 | sendContributors: this.options && !this.options.sendContributors,
|
308 | fileToCommitsMap: !this.handlerParams.fileToCommitsMap
|
309 | }
|
310 | var missing = utils.getMissingProperties(fieldsMap);
|
311 | this.log.error("Cannot add commits indexes, missing fields: " + missing);
|
312 | }
|
313 |
|
314 | FilesMapping.prototype._extractBranches = function (branch, method, file) {
|
315 | if (!branch) {
|
316 | return;
|
317 | }
|
318 |
|
319 |
|
320 |
|
321 | branch.originalFilename = method.originalFilename;
|
322 | file.branches.push(branch);
|
323 | }
|
324 |
|
325 | FilesMapping.prototype._createSlIgnore = function (filename) {
|
326 | var workspacePath = this.options.workspacepath;
|
327 | var lookInParentFolders = !!this.options.instrumentForBrowsers;
|
328 | var slIgnoreScanFilePath = fileAndFolderUtils.findFileRecursively(filename, workspacePath, lookInParentFolders, this.logger);
|
329 | var slIgnoreScan = new IgnoredFilesHandler(slIgnoreScanFilePath, this.options.outputpath, filename, this.log);
|
330 | return slIgnoreScan;
|
331 | }
|
332 |
|
333 | FilesMapping.prototype._mapShortNamesToFiles = function () {
|
334 | this.shortNameToFileMapper = new SlJsInfra.ShortNameToFileMapper(this.logger);
|
335 | var context = this;
|
336 | this.finalFilesData.forEach(function (file) {
|
337 | context.shortNameToFileMapper.addFile(file.physicalPath)
|
338 | });
|
339 | }
|
340 |
|
341 | FilesMapping.prototype._convertUniqueIds = function () {
|
342 | if (!this.shortNameToFileMapper) {
|
343 | this.logger.error("Short name map not intilized.");
|
344 | return;
|
345 | }
|
346 | var context = this;
|
347 | this.finalFilesData.forEach(function (file) {
|
348 | var shortName = context.shortNameToFileMapper.resolveShortNameForFile(file.physicalPath);
|
349 | var converter = new SlJsInfra.UniqueIdConverter(shortName, context.logger);
|
350 | converter.setAndInitFile(file);
|
351 | converter.process();
|
352 | var map = converter.uniqueIdsMap;
|
353 | if (Array.isArray(file.methods)) {
|
354 | file.methods = context._updateUniqueIds(file.methods, map);
|
355 | }
|
356 | if (Array.isArray(file.branches)) {
|
357 | file.branches = context._updateUniqueIds(file.branches, map);
|
358 | }
|
359 | })
|
360 | }
|
361 |
|
362 | FilesMapping.prototype._updateUniqueIds = function (arr, uniqueIdMap) {
|
363 | var newArr = []
|
364 | for (var i = 0; i < arr.length; i++) {
|
365 | var current = arr[i];
|
366 | var oldUniqueId = current.uniqueId;
|
367 | if (uniqueIdMap[oldUniqueId]) {
|
368 | current.uniqueId = uniqueIdMap[oldUniqueId];
|
369 | delete current.parentPosition;
|
370 | newArr.push(current);
|
371 | }
|
372 | }
|
373 | return newArr;
|
374 | }
|
375 |
|
376 | FilesMapping.prototype._getJavaScriptFromMarkupFile = function (fullname, fileContent) {
|
377 | var handler = new MarkupFilesParser(fileContent, fullname, this.logger);
|
378 | handler.extractScriptTags();
|
379 | var scriptTags = handler.scriptTags;
|
380 | if (!scriptTags || scriptTags.length === 0) {
|
381 | this.logger.info("No script tags found in '%s'", fullname);
|
382 | return { hasError: true };
|
383 | }
|
384 | var jsCodeInFile = handler.getAllJsInFile();
|
385 | return jsCodeInFile;
|
386 | };
|
387 |
|
388 | FilesMapping.prototype._isMarkupFile = function (filename) {
|
389 | var dvr = new DefaultValuesResolver();
|
390 | var fileExtensions = dvr.getFileExtensions();
|
391 | var extension = path.extname(filename);
|
392 | var result = (utils.isArrayContainsMarkupFile(fileExtensions) && utils.isMarkupFile(extension));
|
393 | return result;
|
394 | }
|
395 |
|
396 | FilesMapping.prototype._resolveProjectRoot = function () {
|
397 | var projectRoot;
|
398 | if (this.options.projectRoot) {
|
399 | projectRoot = this.options.projectRoot;
|
400 | this.logger.info("Project root set to '%s' from 'sourceRoot'", projectRoot)
|
401 | } else if (this.handlerParams.scmRootDir) {
|
402 | projectRoot = this.handlerParams.scmRootDir;
|
403 | this.logger.info("Project root set to '%s' same as git root", projectRoot)
|
404 | } else {
|
405 | projectRoot = process.cwd();
|
406 | this.logger.info("Project root set to '%s' same as CWD", projectRoot);
|
407 | }
|
408 | return utils.adjustPathSlashes(projectRoot);
|
409 | }
|
410 |
|
411 |
|
412 | FilesMapping.prototype._createActualFile = function (filename, fullFilename) {
|
413 | return new ActualFile(filename, fullFilename)
|
414 | }
|
415 |
|
416 | FilesMapping.prototype.enrichBuildData = function () {
|
417 | var context = this;
|
418 | var fileToSignatureMap = this.createSourcePathToSignatureMap();
|
419 | this.fileSignatures.forEach(function (signature) {
|
420 | if(signature instanceof GeneratedFileSignature){
|
421 | var relevantSourceSignatures = context.extractSourceSignatures(fileToSignatureMap, signature);
|
422 | context.enrichSingleFile(signature, relevantSourceSignatures);
|
423 | }
|
424 | })
|
425 |
|
426 | };
|
427 |
|
428 |
|
429 | FilesMapping.prototype.enrichSingleFile = function (generatedSignature, sourceSignaturesArr) {
|
430 | try {
|
431 | var sourceDataEnricher = new SourceDataEnricher(generatedSignature, sourceSignaturesArr, this.logger);
|
432 | sourceDataEnricher.enrichData();
|
433 | } catch (e) {
|
434 | this.logger.info("Error while enriching build data for '%s'. Error: '%s'", generatedSignature.absoluteFilename, e);
|
435 | globalErrorHandler.setLastError(e)
|
436 | }
|
437 | }
|
438 |
|
439 | FilesMapping.prototype.createSourcePathToSignatureMap = function () {
|
440 | var context = this;
|
441 | var pathToSignatureMap = {};
|
442 | var sourceFilesPath = [];
|
443 | this.fileSignatures.forEach(function (sig) {
|
444 | if(sig instanceof GeneratedFileSignature) {
|
445 | sourceFilesPath = sourceFilesPath.concat(sig.getSourceFilesPath());
|
446 | }
|
447 | });
|
448 | sourceFilesPath = sourceFilesPath.filter(utils.distinctArray);
|
449 | sourceFilesPath = sourceFilesPath.filter(fs.existsSync);
|
450 | sourceFilesPath.forEach(function (filePath) {
|
451 | if (context.shouldParse(filePath)) {
|
452 | var sourceSig = context._generateFileSignature(filePath);
|
453 | if (!sourceSig.hasError) {
|
454 | pathToSignatureMap[filePath] = sourceSig;
|
455 | }
|
456 | } else{
|
457 |
|
458 | }
|
459 | });
|
460 | return pathToSignatureMap;
|
461 | }
|
462 |
|
463 | FilesMapping.prototype.extractSourceSignatures = function(sourceSignatures, generatedSignature){
|
464 | var context = this;
|
465 | var sourcesPath = generatedSignature.getSourceFilesPath();
|
466 | var relevantSignatures = [];
|
467 | sourcesPath.forEach(function (sourcePath) {
|
468 | if(sourceSignatures[sourcePath]) {
|
469 | relevantSignatures.push(sourceSignatures[sourcePath]);
|
470 | }else {
|
471 | context.logger.info("Could not find source signature for '%s'", sourcePath)
|
472 | }
|
473 | })
|
474 | return relevantSignatures;
|
475 | }
|
476 |
|
477 | FilesMapping.prototype.shouldParse = function(absolutePath){
|
478 | var tsFilesPattern = "**/*.ts";
|
479 | var includedPatterns = this.options.includedFiles.concat([tsFilesPattern]);
|
480 | return micromatch.isMatch(absolutePath, includedPatterns);
|
481 | }
|
482 | module.exports = FilesMapping;
|