sl-cia/lib/js_diff/files-mapping.js

var fileAndFolderUtils = require('../file-and-folder-utils');
var SlJsInfra = require('sl-js-infra').SlJsInfra;
var IgnoredFilesHandler = SlJsInfra.IgnoredFilesHandler;
var glob = require("glob");
var path = require('path');
var fs = require('fs');
var ActualFile = require('../file-system/actual-file');
var utils = require('../utils');
var consoleFileLogger = require("../sl-logger.js").consoleFileLogger;
var FileSignature = require('./file-signature.js');
var GeneratedFileSignature = require('./generated-file-signature');
var micromatch = require('micromatch');
var MarkupFilesParser = require('../markup-files-parser');
var DefaultValuesResolver = require('../default-values-resolver');
var EntitiesMapper = require("../entities-mapper");
var SourceDataEnricher = require("./source-data-enricher");
var globalErrorHandler = require('../global-error-handler');
var globalMethodIndexer = require("./globalMethodIndexer");


var BRANCHES_KEY = "branches";
var BRANCH_ELEMENT_TYPE = "branch";
var METHODS_KEY = "methods";
var METHOD_ELEMENT_TYPE = "method";
var METHOD_INDEX_FOR_GLOBAL_BRANCHES = -1;

// TODO: refactor this class and difine the exec responsibility.
function FilesMapping(handlerParams, slMapper, agentEventsController,  logger) {
    this.handlerParams = handlerParams || {};
    this.options = this.handlerParams.buildArguments;
    this.finalOutput = this.handlerParams.finalOutput;
    this.matchedFiles = [];
    this.fileDataByFileName = {};
    this.fileSignatures = [];
    this.handlerParams.processedFiles = [];
    this.erroredFiles = [];         //Relative names
    this.finalFilesData = [];
    this.logger = logger;
    this.log = this._createLogger(logger, "FilesMapping");
    this.agentEventsController = agentEventsController;
    this.slIgnoreScan = this._createSlIgnore(IgnoredFilesHandler.SCAN_IGNORE_FILE_NAME);
    this.slIgnoreGenerated = this._createSlIgnore(IgnoredFilesHandler.GENERATED_IGNORE_FILE_NAME);
    this.projectRoot = this._resolveProjectRoot();
    this.shortNameToFileMapper = null;
    this.slMapper =  slMapper;
    this.counters = {
        methods: 0,
        branches: 0
    };
}

FilesMapping.prototype.process = function () {
    var context = this
    var promise = new Promise(function (resolve) {
        context.log.info("Starting to scan the workspace.");
        context._scanWorkspaceForIncludedFilesAndPopulateFilesArray();
        context.matchedFiles.forEach(function (matchedFile, fileIdx) {
            var isLastFile = (fileIdx == context.matchedFiles.length - 1);
            context._fillFilesData(matchedFile, fileIdx, isLastFile);
        });
        context.fileDataByFileName = {};
        context.enrichBuildData();
        context.fileSignatures.forEach(function (fileSignature) {
            context._mapToServerEntities(fileSignature.createDTO());
        });
        if (context.fileSignatures.length > 0) {
            context.log.info("Processed the following files:");
            context.fileSignatures.forEach(function (file, idx) {
                context.log.info("\t[" + idx + "] Filename: '" + file.relativePath + "'");
            });
        }
        var isSuccess = true;
        if (context.erroredFiles.length) {
            var message = context._getErroredFilesMessage();
            context.log.warn(message);
            isSuccess = false;
        }
        if (SlJsInfra.SlEnvVars.isUseNewUniqueId()) {
            context._mapShortNamesToFiles();
            context._convertUniqueIds();
        }
        context.finalOutput.files = context.finalFilesData;
        context.finalOutput.counters = context.counters;
        return resolve(isSuccess);
    });
    return promise;
}

FilesMapping.prototype._mapToServerEntities = function(fileSignature){
    var context =  this;
    fileSignature.methods.forEach(function(m){
        if(context.slMapper) {
            context.slMapper.mapMethod(m);
        }
        var methodFile = context._getCreateFileData(m.srcData.relativeFilename, m.srcData.absoluteFilename);
        methodFile.methods.push(EntitiesMapper.toServerMethod(m));
        context.counters.methods++;
    })
    fileSignature.branches.forEach(function(b){
        if(context.slMapper) {
            context.slMapper.mapBranch(b);
        }
        // If containing method is ignored (auto generated code) set it to global branch.
        if(globalMethodIndexer.isMethodIgnored(b.enclosingMethodIdx)){
            b.enclosingMethodIdx = METHOD_INDEX_FOR_GLOBAL_BRANCHES;
        }
        var branchesFile = context._getCreateFileData(b.srcData.relativeFilename , b.srcData.absoluteFilename);
        branchesFile.branches.push(EntitiesMapper.toServerBranch(b));
        context.counters.branches++;
    })
}

/* TODO: Split this method into two. For more info: "https://github.com/Sealights/SL.OnPremise.CIA/pull/169"  */
FilesMapping.prototype._scanWorkspaceForIncludedFilesAndPopulateFilesArray = function () {
    var context = this
    context.options.includedFiles.forEach(function (inclusionPattern) {
        var matchedFiles = glob.sync(inclusionPattern,
            {
                cwd: context.options.workspacepath,
                ignore: context.options.excludedFiles
            });

        matchedFiles.forEach(function (matchedFile) {
            if (!context.fileDataByFileName[matchedFile]) {
                context.fileDataByFileName[matchedFile] = true;
                context.matchedFiles.push(matchedFile);
            }
        });
    });
}

FilesMapping.prototype._fillFilesData = function (matchedFile, fileIdx, isLastFile) {
    var absolutePath = path.resolve(this.options.workspacepath, matchedFile);
    absolutePath = utils.adjustPathSlashes(absolutePath);
    var shouldSkipCurrFile = this._shouldSkipFile(absolutePath, matchedFile, fileIdx);
    if (shouldSkipCurrFile)
        return;
    this.handlerParams.processedFiles.push(matchedFile);
    this.log.info('Processing [' + (fileIdx + 1) + '/' + this.matchedFiles.length + '] ' + absolutePath);
    var fileSig = this._generateFileSignature(absolutePath);
    if (!fileSig.hasError) {
        fileSig.methods = this._getIncludedMethods(fileSig);
        fileSig.branches = this._getIncludedBranches(fileSig);
        this.fileSignatures.push(fileSig);
    }
    else {
        this.erroredFiles.push(matchedFile);
    }
}

FilesMapping.prototype._shouldSkipFile = function (fullFilename, relativeFileName, fileIdx) {
    var filePath = fs.statSync(fullFilename);
    if (!filePath.isFile()) {
        this.log.info(relativeFileName + " is not a file.")
        return true;
    }
    // If the file is filtered by the .slignore file:
    if (this.slIgnoreScan.denies(relativeFileName)) {
        this.log.info('Excluded [' + (fileIdx + 1) + '/' + this.matchedFiles.length + '] ' + fullFilename);
        // Ignored by .slignore, skip it.
        return true;
    }
    return false;
}

function createFileSignature(absolutePath, relativePath, fileContent, sourceMaps, opts, scmFilesContainer) {
    if(sourceMaps) {
        return GeneratedFileSignature.create(absolutePath, relativePath, fileContent, sourceMaps, opts, scmFilesContainer);
    }else {
        return FileSignature.create(absolutePath, relativePath, fileContent, sourceMaps, opts, scmFilesContainer);
    }
}

FilesMapping.prototype._generateFileSignature = function (absolutePath) {
    var relativePath = absolutePath.replace(this.projectRoot, "");
    if(relativePath.indexOf("/") === 0) {
        relativePath = relativePath.substring(1);
    }
    var isBranchCoverage = this.handlerParams.cfg.useBranchCoverage || this.options.usebranchcoverage;
    var fileObject = this._createActualFile(relativePath, absolutePath);
    var fileContent = fileObject.getContent();
    var sourceMaps = fileObject.readSourceMaps(this.logger);

    if (this._isMarkupFile(relativePath)) {
        fileContent = this._getJavaScriptFromMarkupFile(absolutePath, fileContent);
    }
    var opts = {
        es6Modules: true,
        babylonPlugins: ["classProperties"].concat(this.options.babylonPlugins),
        projectRoot: this.options.projectRoot,
        scmRootDir: this.handlerParams.scmRootDir,
        isBranchCoverage: this.handlerParams.cfg.useBranchCoverage || this.options.usebranchcoverage
    };
    var extension = path.extname(absolutePath);
    if(extension === ".ts" || extension === ".tsx"){
        opts.babylonPlugins.push("typescript")
    }
    try {
        var fileSignature = createFileSignature(absolutePath, relativePath, fileContent, sourceMaps, opts, this.handlerParams.scmFilesContainer);
        return fileSignature;
    }catch (e) {
        this.logger.error("Failed to create signature for '%s'. Error '%s'" , absolutePath, e.message);
        this.logger.debug(e.stack);
        return {
            hasError: true
        };
    }
};

FilesMapping.prototype._createLogger = function (logger, className) {
    var log = logger ? logger.child({
        className: className
    }) : consoleFileLogger(className);
    return log;
}


FilesMapping.prototype._getIncludedMethods = function (fileSig) {
    if (!fileSig.methods)
        return [];
    return this._getIncludeElements(fileSig, METHODS_KEY, METHOD_ELEMENT_TYPE);;
}



FilesMapping.prototype._getIncludedBranches = function (fileSig) {
    if (!fileSig.branches)
        return []
    return this._getIncludeElements(fileSig, BRANCHES_KEY, BRANCH_ELEMENT_TYPE);

}

FilesMapping.prototype._getIncludeElements = function (fileSig, key, elementType) {
    var includedElements = [];
    var context = this;
    fileSig[key].forEach(function (element) {
        if (fileSig.hasSourceMaps) {
            context._handleGeneratedElement(element, includedElements, elementType);
        } else {
            includedElements.push(element);
        }
    });
    return includedElements;
}

FilesMapping.prototype._handleGeneratedElement = function (element, elementsArr, elementType) {
    if (this.slIgnoreGenerated.accepts(element.srcData.relativeFilename)) {
        elementsArr.push(element);
    } else {
        if(elementType === METHOD_ELEMENT_TYPE){
            globalMethodIndexer.setIgnoredMethod(element.idxInMapping);
        }
        this.log.info('Rejecting ' + elementType + ' from file: ' + element.srcData.relativeFilename);
    }
}

FilesMapping.prototype._getErroredFilesMessage = function () {
    return "The following files failed to be parsed and will not be sent to the server:\n" +
        this.erroredFiles.join('\n') +
        "\n\t(Hint: if the file use the es6 module syntax, pass the \"--es6Modules\" switch)";
}

FilesMapping.prototype._getCreateFileData = function (filename, absoluteFileName) {
    if (!this.fileDataByFileName[filename]) {
        var fileData = this._createNewFileData(filename, absoluteFileName);
        this.fileDataByFileName[filename] = fileData;
        this.finalFilesData.push(fileData);
    }
    return this.fileDataByFileName[filename];
}

FilesMapping.prototype._createNewFileData = function (filename, absoluteFileName) {
    var gitPath = this._resolvePhysicalPath(absoluteFileName) || filename;
    var newFile = {
        logicalPath: filename,
        physicalPath: gitPath, //Relative path to git folder
        methods: [],
        lines: [],
        branches: []
    };

    if (this._shouldAddCommitsArray()) {
        newFile.commitIndexes = this.handlerParams.fileToCommitsMap[gitPath] || [];
    } else {
        this._logCommitsArrayNotFound();
    }

    return newFile;
}

FilesMapping.prototype._resolvePhysicalPath = function (absoluteFileName) {
    if (!this.handlerParams.scmRootDir) {
        this.logger.debug("scmRootDir not found");
        return;
    }
    absoluteFileName = utils.adjustPathSlashes(absoluteFileName);
    var scmRootDir = utils.adjustPathSlashes(this.handlerParams.scmRootDir);
    var physicalPath = absoluteFileName.replace(scmRootDir, "");
    if (physicalPath.indexOf("/") === 0) {
        physicalPath = physicalPath.substring(1);
    }
    return physicalPath;
};

FilesMapping.prototype._shouldAddCommitsArray = function () {
    return this.options && this.options.sendContributors && this.handlerParams.fileToCommitsMap;
}

FilesMapping.prototype._logCommitsArrayNotFound = function () {
    var fieldsMap = {
        opts: !this.options,
        sendContributors: this.options && !this.options.sendContributors,
        fileToCommitsMap: !this.handlerParams.fileToCommitsMap
    }
    var missing = utils.getMissingProperties(fieldsMap);
    this.log.error("Cannot add commits indexes, missing fields: " + missing);
}

FilesMapping.prototype._extractBranches = function (branch, method, file) {
    if (!branch) {
        return;
    }
    //TODO: Hack! Currently the branches which we collect are inside method.
    //When we'll support global branches, we should get the "original file name"
    //in the same time we create the branch data object.
    branch.originalFilename = method.originalFilename;
    file.branches.push(branch);
}

FilesMapping.prototype._createSlIgnore = function (filename) {
    var workspacePath = this.options.workspacepath;
    var lookInParentFolders = !!this.options.instrumentForBrowsers;
    var slIgnoreScanFilePath = fileAndFolderUtils.findFileRecursively(filename, workspacePath, lookInParentFolders, this.logger);
    var slIgnoreScan = new IgnoredFilesHandler(slIgnoreScanFilePath, this.options.outputpath, filename, this.log);
    this.submitIgnorePatternToCockpit(slIgnoreScan, filename)
    return slIgnoreScan;
}

FilesMapping.prototype._mapShortNamesToFiles = function () {
    this.shortNameToFileMapper = new SlJsInfra.ShortNameToFileMapper(this.logger);
    var context = this;
    this.finalFilesData.forEach(function (file) {
        context.shortNameToFileMapper.addFile(file.physicalPath)
    });
}

FilesMapping.prototype._convertUniqueIds = function () {
    if (!this.shortNameToFileMapper) {
        this.logger.error("Short name map not intilized.");
        return;
    }
    var context = this;
    this.finalFilesData.forEach(function (file) {
        var shortName = context.shortNameToFileMapper.resolveShortNameForFile(file.physicalPath);
        var converter = new SlJsInfra.UniqueIdConverter(shortName, context.logger);
        converter.setAndInitFile(file);
        converter.process();
        var map = converter.uniqueIdsMap;
        if (Array.isArray(file.methods)) {
            file.methods = context._updateUniqueIds(file.methods, map);
        }
        if (Array.isArray(file.branches)) {
            file.branches = context._updateUniqueIds(file.branches, map);
        }
    })
}

FilesMapping.prototype._updateUniqueIds = function (arr, uniqueIdMap) {
    var newArr = []
    for (var i = 0; i < arr.length; i++) {
        var current = arr[i];
        var oldUniqueId = current.uniqueId;
        if (uniqueIdMap[oldUniqueId]) {
            current.uniqueId = uniqueIdMap[oldUniqueId];
            delete current.parentPosition;
            newArr.push(current);
        }
    }
    return newArr;
}

FilesMapping.prototype._getJavaScriptFromMarkupFile = function (fullname, fileContent) {
    var handler = new MarkupFilesParser(fileContent, fullname, this.logger);
    handler.extractScriptTags();
    var scriptTags = handler.scriptTags;
    if (!scriptTags || scriptTags.length === 0) {
        this.logger.info("No script tags found in '%s'", fullname);
        return { hasError: true };
    }
    var jsCodeInFile = handler.getAllJsInFile();
    return jsCodeInFile;
};

FilesMapping.prototype._isMarkupFile = function (filename) {
    var dvr = new DefaultValuesResolver();
    var fileExtensions = dvr.getFileExtensions();
    var extension = path.extname(filename);
    var result = (utils.isArrayContainsMarkupFile(fileExtensions) && utils.isMarkupFile(extension));
    return result;
}

FilesMapping.prototype._resolveProjectRoot = function () {
    var projectRoot;
    if (this.options.projectRoot) {
        projectRoot = this.options.projectRoot;
        this.logger.info("Project root set to '%s' from 'sourceRoot'", projectRoot)
    } else if (this.handlerParams.scmRootDir) {
        projectRoot = this.handlerParams.scmRootDir;
        this.logger.info("Project root set to '%s' same as git root", projectRoot)
    } else {
        projectRoot = process.cwd();
        this.logger.info("Project root set to '%s' same as CWD", projectRoot);
    }
    return utils.adjustPathSlashes(projectRoot);
}

//This method was added for testing.
FilesMapping.prototype._createActualFile = function (filename, fullFilename) {
    return new ActualFile(filename, fullFilename)
}

FilesMapping.prototype.enrichBuildData = function () {
    var context = this;
    var fileToSignatureMap = this.createSourcePathToSignatureMap();
    this.fileSignatures.forEach(function (signature) {
        if(signature instanceof GeneratedFileSignature){
            var relevantSourceSignatures = context.extractSourceSignatures(fileToSignatureMap, signature);
            context.enrichSingleFile(signature, relevantSourceSignatures);
        }
     })

};


FilesMapping.prototype.enrichSingleFile = function (generatedSignature, sourceSignaturesArr) {
    try {
        var sourceDataEnricher = new SourceDataEnricher(generatedSignature, sourceSignaturesArr, this.logger);
        sourceDataEnricher.enrichData();
    } catch (e) {
        this.logger.info("Error while enriching build data for '%s'. Error: '%s'", generatedSignature.absoluteFilename, e);
        globalErrorHandler.setLastError(e)
    }
}

FilesMapping.prototype.createSourcePathToSignatureMap = function () {
    var context = this;
    var pathToSignatureMap = {};
    var sourceFilesPath = [];
    this.fileSignatures.forEach(function (sig) {
        if(sig instanceof GeneratedFileSignature) {
            sourceFilesPath = sourceFilesPath.concat(sig.getSourceFilesPath());
        }
    });
    sourceFilesPath = sourceFilesPath.filter(utils.distinctArray);
    sourceFilesPath = sourceFilesPath.filter(fs.existsSync);
    sourceFilesPath.forEach(function (filePath) {
        if (context.shouldParse(filePath)) {
            var sourceSig = context._generateFileSignature(filePath);
            if (!sourceSig.hasError) {
                pathToSignatureMap[filePath] = sourceSig;
            }
        } else{
            //TODO: Report do OBD
        }
    });
    return pathToSignatureMap;
}

FilesMapping.prototype.extractSourceSignatures = function(sourceSignatures, generatedSignature){
    var context = this;
    var sourcesPath = generatedSignature.getSourceFilesPath();
    var relevantSignatures = [];
    sourcesPath.forEach(function (sourcePath) {
        if(sourceSignatures[sourcePath]) {
            relevantSignatures.push(sourceSignatures[sourcePath]);
        }else {
            context.logger.info("Could not find source signature for '%s'", sourcePath)
        }
    })
    return relevantSignatures;
}

FilesMapping.prototype.shouldParse = function(absolutePath){
    var tsFilesPattern = "**/*.ts";
    var includedPatterns = this.options.includedFiles.concat([tsFilesPattern]);
    return  micromatch.isMatch(absolutePath, includedPatterns);
}

FilesMapping.prototype.submitIgnorePatternToCockpit = function(ignoredFilesHandler, filename){
    var formattedPattern = ignoredFilesHandler.getFormattedPattern();
    if(!formattedPattern){
        return;
    }
    var ignoreFile = ignoredFilesHandler.isUsingNycConfig ? "NYC configuration" : filename;
    var messaage = "Using the following ignore pattern from " + ignoreFile + ": " + ignoredFilesHandler.getFormattedPattern();
    this.agentEventsController.submitMessage(messaage);
}
module.exports = FilesMapping;