sl-cia/lib/build-diff-process.js

var Promise = require('promise');
var FilesMapping = require("./js_diff/files-mapping");
var consoleFileLogger = require("./sl-logger.js").consoleFileLogger;
var version = require("../package.json").version;
var utils = require('./utils.js');
var os = require("os");
var DefaultValuesResolver = require('./default-values-resolver');
var fileAndFolderUtils = require("./file-and-folder-utils");
var BrowserInstrumenter = require('./instrumentation/browser-instrumenter');
var dvr =  new DefaultValuesResolver();
var SlMapper = require("./js_diff/sl-mapper").SlMapper;
var ScmFilesContainer = require("./scm/scm-files-container");
var SlJsInfra = require("sl-js-infra").SlJsInfra;
var CiaEnvVars = SlJsInfra.SlEnvVars.CIA;
var path = require("path");
var fs = require("fs");

function BuildDiffProcess(cfg, backendProxy, sourceConrolProviders, parsedDependencies, logger) {
    var log = logger ? logger.child({
        className: "BuildDiffProcess"
    }) : consoleFileLogger("BuildDiffProcess");

    function createBuildDiffPromiseChain(buildArguments) {
        var handlerParams = {
            buildArguments: buildArguments,
            cfg: cfg,
            finalOutput: {
                meta: createMetaTag(buildArguments, cfg),
                version: "2.0",
                files: [],
                dependencies: parsedDependencies || []
            }
        };

        var scmInstance;

        if (buildArguments.workspacepath) {
            var scmType = sourceConrolProviders[buildArguments.scm];
            if (scmType) {
                scmInstance = new scmType(cfg, buildArguments.workspacePath);
                handlerParams.finalOutput.meta.scm = buildArguments.scm;
                log.info("Created an scmInstance of type '" + scmType.name + "'. Searching for a local repository at '" + buildArguments.workspacepath + "'.");
            }else{
                log.warn("Unsupported scmType: '" + scmType + "'.");
            }
        }

        if (!buildArguments.commit && scmInstance && !scmInstance.detectCommitVersion) {
            return Promise.reject("'commit' command-line argument is required, since automatic detection by the " + buildArguments.scm + " SCM is not supported");
        }

        var promiseChain = Promise.resolve();

        if (buildArguments.workspacepath) {

            //2. If there is a mapping, get the diffs
            if (scmInstance) {
                //2.1 Get repository url from the SCM
                promiseChain = promiseChain.then(function (mapping) {
                    log.info("Getting repository URL.");
                    return getRepositoryUrl(handlerParams, scmInstance);
                });

                //2.2 Get all contributors from the scm
                if(buildArguments.sendContributors){
                    promiseChain = promiseChain.then(function(mapping){
                        log.info("Getting contributors.");
                        return getContributors(handlerParams, scmInstance);
                    });
                }

                //2.3 Get branch history from the SCM
                promiseChain = promiseChain.then(function (mapping) {
                    log.info("Loading branch history.");
                    return loadBranchHistory(handlerParams, scmInstance);
                });

                //2.4 Get all commits effected each file
                if(buildArguments.sendContributors){
                    promiseChain= promiseChain.then(function(mapping){
                        log.info("Getting commits per file.");
                        return getCommitsPerFile(handlerParams, scmInstance);
                    });
                }
                //2.5 if commit was not specified, automatically detect it
                if (!buildArguments.commit) {
                    promiseChain = promiseChain.then(function () {
                        log.info("Couldn't find a 'commit' on the 'buildArguments'. Trying to detect the commit version.");
                        return scmInstance.detectCommitVersion(handlerParams, log).then(function (commit) {
                            if (commit) {
                                handlerParams.finalOutput.meta.commit = commit;
                            }
                        });
                    });
                };

                //2.6 Find repository root directory
                promiseChain = promiseChain.then(function () {
                    return scmInstance.getRootDirectory(handlerParams).then(function (rootDir) {
                        handlerParams.scmRootDir = rootDir;
                    })
                })

                //2.6 Get list of all files in git
                promiseChain = promiseChain.then(function () {
                    return scmInstance.getFiles(handlerParams).then(function (scmFiles) {
                        if(scmFiles && Array.isArray(scmFiles)){
                          var scmFilesContainer = new ScmFilesContainer(logger);
                           scmFiles.forEach(function (file) {
                               scmFilesContainer.addFile(file);
                           })
                           handlerParams.scmFilesContainer = scmFilesContainer;
                        }
                    })
                })
            }

            //3. Call JS mapping handler
            var slMapper = null;
            if(buildArguments.instrumentForBrowsers) {
                 slMapper = new SlMapper(CiaEnvVars.getSlMappingPath(), backendProxy, buildArguments.buildsessionid, logger);
            }
            promiseChain = promiseChain.then(function () {
                log.info("Handling files and sending information in the new Build Scanner Format (with UniqueID).");
                var filesMapping = new FilesMapping(handlerParams, slMapper, log);
                return filesMapping.process();
            });
        };

        //4. Submit the final output to the server
        promiseChain = promiseChain.then(function () {
            if (buildArguments.instrumentationOnly) {
                log.info("No need to submit the files to server as the 'instrumentationOnly' flag is on.");
                return Promise.resolve();
            } else {
                //TODO: [SLDEV-4708] When the call to the server fails, the agent is "silent" and not showing any error message.
                return backendProxy.submitBuildMapping(handlerParams.finalOutput);
            }
        });

         //5. Print the counters
         promiseChain = promiseChain.then(function () {
            var mapping = handlerParams.finalOutput;
            if(mapping != null && mapping.counters !=null  && mapping.counters.methods != null && mapping.counters.branches != null){
                console.log("Reported " + mapping.counters.methods + " methods and " + mapping.counters.branches + " branches in " + mapping.files.length + " files.");
            }
        });

        //6. Create instrumented browser files.
        if (buildArguments.instrumentForBrowsers) {
            promiseChain = promiseChain.then(function () {
                return submitSlMapping(promiseChain, slMapper, logger);
        })
            promiseChain = promiseChain.then(function () {

                if (handlerParams.finalOutput && handlerParams.finalOutput.files) {
                    delete handlerParams.finalOutput.files;
                }
                var promise = new Promise(function (resolve, reject) {
                    var resolveWithoutHash = true;
                    var delayShutdownInSeconds =buildArguments.delayShutdownInSeconds || 30;
                    var downloadAgent = buildArguments.downloadAgent != null ? buildArguments.downloadAgent : true;

                    var instrumentationConfig = {
                            appName: buildArguments.appname,
                            build: buildArguments.build,
                            customerId: cfg.customerId,
                            branch: buildArguments.branch,
                            sourceRoot: buildArguments.workspacepath,
                            files: getProcessedFiles(buildArguments, handlerParams.processedFiles, logger),
                            outputPath: buildArguments.outputpath,
                            copyAllFilesToOutput: buildArguments.copyAllFilesToOutput !== false,
                            buildsessionid: buildArguments.buildsessionid,
                            esModules: buildArguments["es-modules"],
                            server: handlerParams.cfg.server,
                            token: cfg.token,
                            instrumentationType: "browser",
                            resolveWithoutHash: resolveWithoutHash,
                            delayShutdownInSeconds: delayShutdownInSeconds,
                            downloadAgent: downloadAgent,
                            workspacepath: buildArguments.workspacepath,
                            folderToInstrument: buildArguments.folderToInstrument,
                            isInstrumentOriginals: !!buildArguments.folderToInstrument,
                            slMappingPath: CiaEnvVars.getSlMappingPath(),
                            slMappingUrl: CiaEnvVars.getSlMappingUrl() ||CiaEnvVars.getSlMappingPath()

                        };

                    log.info("'instrumentForBrowsers' flag is on. Starting instrumentation. Instrumentation config: ", JSON.stringify(instrumentationConfig));
                    var callback =  function (err) {
                        if (err){
                            log.info("Finished the instrumentation with an error. Error:", err);
                            reject(err);
                        }
                        else{
                            log.info("Finished the instrumentation successfully");
                            return resolve();
                        }
                    };

                    runBrowserInstrumentation(log, instrumentationConfig, callback, slMapper);
                });
                return promise;
            });
        }

        promiseChain = promiseChain.then(function () {
            return Promise.resolve();
        }).catch(function (err) {
            return Promise.reject(err);
        });

        return promiseChain;
    }

    this.run = function run(buildArguments) {
        return createBuildDiffPromiseChain(buildArguments);
    }

    function runBrowserInstrumentation(log, instrumentationConfig, callback, slMapper){
        log.info("Running browser instrumentation.");
        instrumentationConfig.extensionsToInstrument = dvr.getFileExtensions();
        var bi = new BrowserInstrumenter(instrumentationConfig, log, slMapper);
        bi.instrument(callback);
    }

    function loadBranchHistory(handlerParams, scmInstance) {
        return scmInstance.getBranchHistory(handlerParams, log).then(function (gitLog) {
            if (gitLog && gitLog.commitHistory && gitLog.commitHistory.length && gitLog.commitLog && gitLog.commitLog.length) {
                log.info('Branch history loaded (' + gitLog.commitHistory.length + ' commits)');
                handlerParams.finalOutput.meta.history = gitLog.commitHistory;
                handlerParams.finalOutput.meta.commitLog = gitLog.commitLog;
            } else {
                logBranchHistoryNotFound(gitLog, log);
            }
            return loadCommitIndexMap(gitLog, handlerParams, log);
        });
    }

    function getRepositoryUrl(handlerParams, scmInstance) {
        return scmInstance.getRepositoryUrl(handlerParams, log).then(function (repositoryUrl) {
            if (repositoryUrl) {
                log.info('Repository Url loaded (' + repositoryUrl + ')');
                handlerParams.finalOutput.meta.repositoryUrl = repositoryUrl;
            }
            return repositoryUrl;
        });
    }

    function getContributors(handlerParams, scmInstance) {
        return scmInstance.getContributors(handlerParams, log).then(function (contributorsLog) {
            if (contributorsLog && contributorsLog.contributors && contributorsLog.emailToIndexMap) {
                log.info('Contributors list loaded (' + contributorsLog.contributors.length + ' contributors)');
                handlerParams.finalOutput.meta.contributors = contributorsLog.contributors;
                handlerParams.emailToIndexMap = contributorsLog.emailToIndexMap;
            }else{
                logContributorsNotFound(contributorsLog, log);
            }
            return contributorsLog;
        })
    }

    function getCommitsPerFile(handlerParams, scmInstance){
        return scmInstance.getCommitsPerFile(handlerParams, log).then(function(fileToCommitsMap){
            if(fileToCommitsMap != null){
                log.info('File commit map loaded (' + Object.keys(fileToCommitsMap).length + ' records)');
                handlerParams.fileToCommitsMap = fileToCommitsMap;
            }else{
                log.error("Commits map is null or undefined.")
            }
            return fileToCommitsMap;
        })
    }
}

function logContributorsNotFound(contributorsLog, log) {
    var fieldsMap = {
        contributorsLog: !contributorsLog,
        contributors: contributorsLog && !contributorsLog.contributors,
        emailToIndexMap: contributorsLog && !contributorsLog.emailToIndexMap
    };
    var missing = utils.getMissingProperties(fieldsMap)
    log.error("Contributors list not loaded, missing fields: " + missing);
}


function logBranchHistoryNotFound(gitLog, log) {
    var fieldsMap = {
        gitLog: !gitLog,
        commitHistory: gitLog && !gitLog.commitHistory,
        commitHistoryLength: gitLog && gitLog.commitHistory && !gitLog.commitHistory.length,
        commitLog: gitLog && !gitLog.commitLog,
        commitLogLength: gitLog && gitLog.commitLog && !gitLog.commitLog.length
    };
    var missing = utils.getMissingProperties(fieldsMap)
    log.error("Branch history not loaded, missing fields: " + missing);
}

function clone(o) {
    //if (!o) return o; //Unreachable
    return JSON.parse(JSON.stringify(o));
}

var getProcessedFiles =function (buildArguments, processedFiles, logger) {
    var folderToScan = buildArguments.folderToInstrument || buildArguments.workspacepath;
       return fileAndFolderUtils.scanDirRecursively(folderToScan, logger)
}

var loadCommitIndexMap = function (gitLog, handlerParams, log) {
    if(!handlerParams.buildArguments.sendContributors){
        return gitLog;
    }
    if (gitLog && gitLog.commitToIndexMap) {
        handlerParams.commitToIndexMap = gitLog.commitToIndexMap;
    } else {
        log.error("Commit log index map not loaded")
    }
    return gitLog;
};

function createMetaTag(buildArguments, cfg){
    return {
        "generated": new Date().valueOf(),
        "build": buildArguments.build,
        "commit": buildArguments.commit,
        "customerId": cfg.customerId,
        "appName": buildArguments.appname,
        //"technologies": "nodejs",
        "technology": "nodejs",
        "branch": buildArguments.branch,
        "environment": {
            agentVersion: version,
            agentType: "nodeJsBuildScanner",
            processArgv:process.argv,
            machineName: os.hostname(),
            platform: os.platform(),
            os: os.type(),
            osVersion: os.release(),
            arch: os.arch(),
            processId: process.pid,
            ipAddress: clone(os.networkInterfaces()),
            dependencies: clone(process.versions),
            runtime: process.version
        },
        "authors": buildArguments.author,
        "logsUrl": buildArguments.logsUrl,
        "jobName": buildArguments.jobName,
        "moduleName":  buildArguments.uniqueModuleId, //Added for backward compatibility incase the server needs it.
        "uniqueModuleId": buildArguments.uniqueModuleId,
        "configurationData": buildArguments,
        "scmProvider": buildArguments.scmProvider,
        "scmVersion": buildArguments.scmVersion,
        "scmBaseUrl": buildArguments.scmBaseUrl
    }
}

function submitSlMapping(promiseChain, slMapper, logger){
    return new Promise(function (resolve, reject) {
        if(!CiaEnvVars.getSlMappingPath()) {
            slMapper.postToServer().then(function () {
                resolve()
            }, function (err) {
                reject(err);
            });
        }else {
            logger.info("Sl mapping path was set to '%s', skipping post to server");
            resolve();
        }

    })
}


module.exports.BuildDiffProcess = BuildDiffProcess;
module.exports.sourceControlProviders = require("./source-control-providers.js");