hp_pricebot/index.js

/*
HOTELPARTNER
Autor:          HotelPartner WDS
Author's note:  “Clean code always looks like it was written by someone who cares.” ― Robert C. Martin,

Date:           16.08.2017
Version:        2.0
Notes:          Make sure that the hostname is set correct!
===============================

 */


//HP LIBARIES
const _HP           = require("./classHotels.js");
const _CONTROLLER   = require("./bot_controller.js");

const slave         = _CONTROLLER.os.hostname().replace('slave','');
const socket        = _CONTROLLER.io.connect('http://192.168.254.89:88/', {
    query:'slave='+slave,
    reconnect: true
});

//Global Variables

let currentip       = '0.0.0.0';
let errjobcounter   = 0;
let ipischanging    = false;
let client;
let mobile;
let api;
let hostip;
let hostport;


// ##ENUMS
const ADULT = {
    ERROR: 0,
    ONE  : 1,
    TWO  : 2
};
const JOB   = {
    DONE:1,
    CITY_ID_LEFT:0
};

// ## PROTOTYPING
Number.prototype.isActive           = function () {
    return(this!=-1)
};
Object.prototype.prepareNextRequest = function (callback) {

    const arg = this;

    //check if job is valid
    if(arg.cityid[0]=='' || !arg.cityid[0] || !arg.dates[0]){
        hp_bot.job.getNewJob(true)
        // _CONTROLLER.reboot();
        return null;
    }

    //repace all spaces with +
    let city        = arg.name.split(' ').join('+');
    let countcity   =  arg.cityid.length-1;
    let countdate   =  arg.dates.length-1;


    callback(arg.dates[countdate], city ,arg.cityid[countcity]);
};


const hp_bot = {
    "constructor": function JOB(_api=-1,_mobile=-1,_client='WDS',_host='192.168.56.1',_port='88') {

        //Set Variables
        api      = _api;
        mobile   = _mobile;
        client   = _client;
        hostip   = _host;
        hostport = _port;


        //START SERVER
        _CONTROLLER.start();
        //Tell Host we are online
        socket.emit('slaves',slave)


        setInterval(()=>{
            socket.emit(slave,'says hello')
        },1000)
    },
    "init":        function init() {
        setTimeout( async function start(){

            //get job
            let job = await hp_bot.job.searchJob()
          currentip = await _CONTROLLER.showExternalIP(true);
            // here we should do a foreach to get each cityid and each Date
            async function startForeach()
            {
                job.prepareNextRequest(async function (startdate,cityname,cityid) {

                    var bot = -5;

                    if(mobile.isActive())
                        bot =  await hp_bot.bot.mobile(job,startdate,cityname,cityid);

                    else if(api.isActive())
                        bot = await hp_bot.bot.api(job,startdate,cityname,cityid)

                    else
                        bot = await hp_bot.bot.standart(job,startdate,cityname,cityid);

                    //get next job

                    if( bot == JOB.DONE )
                    {

                        console.log("=> JOB done <=")
                        _HP.hotels.Booking = []

                        hp_bot.job.saveJob([])
                        hp_bot.job.nextJob(hp_bot.init)
                    }

                   else if( bot == JOB.CITY_ID_LEFT)
                    {
                        setTimeout(async function(){
                            console.log("== next => ")
                            job = await hp_bot.job.getOldJob();
                            console.log(job)
                            startForeach();
                        },1000)

                    }

                })
            }

            startForeach();

        },1800)
    },
    job:    {
        'getNewJob'    :       function getNewJob(rebootOnError=true)      {
            console.log("");
            console.log("GETTING NEW JOB FROM DB... CLIENT:" +client );

            let data = "";
            let clientVersion = '2217'

            return new Promise(resolve=>{

                //check clientverison and callback to _clientVersionCallB
                // with the correct clientVersion
                function makeRequest(clientversion){
                    console.log('clientversion '+clientversion)

                    _CONTROLLER.http.get('http://www.yieldagent.ch/relay/cityov/requestJobV8.php?version='+clientVersion+'&client='+client, function(res) {
                        if (res.statusCode >= 200 && res.statusCode < 400)
                        {
                            res.on('data', function(data_) { data += data_.toString(); });
                            res.on('end', function() {
                                _CONTROLLER.parseXML(data, function (err, result) {

                                    try
                                    {
                                        if (result)
                                        {
                                            //set newclient version
                                            clientVersion = result.Request.$.clientVersion;

                                            //save client version
                                            _CONTROLLER.fs.writeFile('./clientVersion.txt', clientVersion, 'utf8', function readFileCallback(err, data) {
                                                if (err)
                                                    console.log(err);

                                                else
                                                    console.log("new Client Version "+clientVersion)

                                            });

                                            //Parse XML
                                            let ids     = [];
                                            let city    = result.Request.CityConfig[0].URLArguments[0];
                                                city    = city.split(';city=');
                                            //remove unnecessary elements
                                            city.shift();
                                            city.forEach(t => {
                                                t = t.substr(0, t.indexOf(';'))
                                                ids.push(t)
                                            });

                                            //build json
                                            let job = {
                                                "id": result.Request.CityConfig[0].CityID[0],
                                                "cityid": ids,
                                                "name": result.Request.CityConfig[0].CityName[0],
                                                "dates": result.Request.Dates[0].Date,
                                                "backup": result.Request.Dates[0].Date,
                                                "job": result.Request.JobID[0],
                                                "los": result.Request.$.los
                                            };

                                            //save job
                                            hp_bot.job.saveJob(job);

                                            //check for errors in JOB
                                            if(!result.Request.Dates[0].Date){
                                                hp_bot.general.errorHandler(job,1,_CONTROLLER.time().format('YYYY-MM-DD'),job.name,job.cityid,'{"error":"no date"}')
                                                hp_bot.job.saveJob([]);
                                            }
                                            if(!result.Request.CityConfig[0].CityName[0]){
                                                hp_bot.general.errorHandler(job,1,moment().format('YYYY-MM-DD'),'null',0,'{"error":"city"}')
                                                hp_bot.job.saveJob([]);
                                            }

                                            resolve(job)
                                        }
                                        if(err)
                                        {
                                            console.log(err)
                                            setTimeout(()=>{
                                                _CONTROLLER.reboot();
                                            },680000)
                                        }
                                    }
                                    catch (TypeError)
                                    {
                                        //make request with the new clienversion
                                        setTimeout( makeRequest(clientversion),8000);

                                        //LOG
                                        console.log("JOB parsing error")
                                        console.log(result.Request)
                                        console.log(" ")
                                        console.log("getting new jeb with new clientVersion:" +clientVersion)


                                    }

                                });
                            });
                        }
                    }).on('error',
                        async function(err) {

                            //Get current bot version
                            let version = await _CONTROLLER.version(false);

                            //Handle error
                            console.log("Error while fetching new Job");
                            console.log(err);

                            //request new ip
                            let hosturl = 'http://'+hostip+':'+hostport;
                            _CONTROLLER.fetch(hosturl+'/newip?gateway='+slave+'&client='+client+'&job=-1&version='+version);

                            // reboot if needed
                            setTimeout(()=>
                            {
                                if(errjobcounter >=3 && rebootOnError == true)
                                    _CONTROLLER.fetch(hosturl+'/reboot?gateway='+slave+'&client='+client+'&job=error&version='+version);

                                else
                                    _CONTROLLER.fetch(hosturl+'/newip?gateway='+slave+'&client='+client+'&version='+version).then(newIP=>{
                                        hp_bot.job.getNewJob();
                                    })

                                errjobcounter++;
                            },30000)
                        }
                    );
                }

                //Read clientsversion
                _CONTROLLER.fs.readFile('./clientVersion.txt','utf8', function read(err, data)
                {
                    if (err)
                    {
                        console.log('no file: clientVersion.txt')
                        makeRequest(clientVersion);
                        return;
                    }
                    else
                    {
                        clientVersion = data;
                        makeRequest(data)
                    }

                });
            })

        },
        'getOldJob'    :       function getOldJob()                        {
            console.log("searching for old job");
            try{
                //Look for a backup file
                const backup = JSON.parse(_CONTROLLER.fs.readFileSync('./backup.json', 'utf8'));

                //Check if the file contains valid data
                const checkforid = backup.cityid[0];
                const checkfordate = backup.dates[0];
                return backup;
            }
            catch (TypeError)
            {
                console.log("no Job found..")
                return null
            }
        },
        'searchJob'    :      async function () {
            var job    = hp_bot.job.getOldJob();

            if(!job)
                job    = await hp_bot.job.getNewJob();
            else
                console.log("BACKUP JOB FOUND");

            return job;
        },
        'updateOldJob' :       function (date,cityid,resetdate=false)      {
            return new Promise(resolve=>{

            try{

                let backup          =  JSON.parse(_CONTROLLER.fs.readFileSync('./backup.json', 'utf8'));
                const indx          =  backup.dates.indexOf(date);

                //Update dates
                if (indx > -1 && backup.dates.length>=1)
                {
                    console.log("removing date")
                    backup.dates.splice(indx, 1);
                    console.log( " ")
                    console.log("****dates left: "+ backup.dates.length)
                    console.log( " ")
                }

                //Update cityid
                if( backup.dates.length==0)
                {
                    console.log("removing city id")
                    const indx2= backup.cityid.indexOf(cityid)
                    if (indx2 > -1){
                        backup.cityid.splice(indx2, 1);
                        backup.dates = backup.backup;
                    }

                    if(backup.cityid.length==0){
                        console.log("Clearing JOB")
                        hp_bot.job.saveJob([]).then(r=>{
                            resolve(JOB.DONE)
                        });

                    }


                }

                //Save the new Backup
                hp_bot.job.saveJob(backup).then(r=>{
                    resolve(JOB.CITY_ID_LEFT)
                });

            }
            catch (TypeError)
            {
                console.log('!!! JOB NOT SAVED !!!')
                console.log(TypeError)
                resolve(false)
            }
            })
        },
        'nextJob'      : async function nextJob(callback)                  {
            console.log("get new Job")
            const newjob = await hp_bot.job.getNewJob();
            callback(newjob);
        },
        'saveJob'      :       function (json)                             {
            if(!json)return;
            return new Promise(resolve=>{


            console.log(" ")
            console.log("Saving Job | backup.json");
            _CONTROLLER.fs.writeFile('./backup.json', JSON.stringify(json), 'utf8', function readFileCallback(err, data) {
                if (err)
                    console.log(err);
                else
                    console.log("JOB is SAVED")

                console.log(" ")
                resolve(true)
            })
            });
        },
        'sendJobToHost':       function (json_job,httpOrSocket='socket')   {
            return new Promise(resolve => {
                if (httpOrSocket == 'socket') {
                    socket.emit('jobdone', JSON.stringify(json_job), function (response) {
                        console.log(response);
                        resolve(response)
                    });
                }
            })
        },
    },
    bot:    {
        "mobile"      : async function startMobileJob  (arg,startdate,cityname,cityid,API=null)              {

            console.log("->Fetching Mobile Version");
            const end = _CONTROLLER.time(startdate,'YYYY-MM-DD').add(1, 'day').format('YYYY-MM-DD');
            let counterror  =  0;
            return new Promise(resolve=>{
                 let adult =1;

                 async function startFetch(adult) {
                     console.log(startdate,cityname,cityid,adult);

                        const xmlresult = await hp_bot.general.getXMLData(adult,startdate,end,cityname,cityid);

                        const isvalid   =  hp_bot.general.validateXmlResult(xmlresult,arg,adult,startdate,cityname,cityid);
                        if(!isvalid)return;

                        //Parse Data
                        const parsed = await _HP.methods.Booking.parsemobile(xmlresult.result,adult,startdate,end,cityname,cityid);

                        if(parsed==ADULT.ONE){
                            startFetch(2)
                        }
                        if(parsed==ADULT.TWO)
                        {
                            counterror = 0;
                            const json = await _HP.methods.general.generateJson(_HP.hotels.Booking, arg.job, arg.id, startdate, slave);

                            if(API)
                            {
                                API(json)
                                resolve(json)
                            }
                            else
                            {
                                const answer = await hp_bot.job.sendJobToHost(json, 'socket');
                                if (answer == 'OK')
                                {
                                    console.log("SENDED: OK")
                                   let _status = await hp_bot.job.updateOldJob(startdate, cityid);
                                    resolve(_status)
                                }
                                else {
                                    startFetch(adult);
                                }
                            }

                        }
                        if(parsed==ADULT.ERROR){

                            console.log("ip got blocked")

                            //check if there is a error with the url
                            if(4<counterror)
                            {
                                let message;
                                try{
                                    message = allresult.errors[0].message;
                                    hp_bot.general.errorHandler(arg,adult,startdate,cityname,cityid,message,'Mobile Parser',true)
                                }catch (TypeError)
                                {

                                    if(JSON.stringify(allresult).indexOf('in_city_translation')>=0)
                                    {
                                        message = {'error':'in_city_translation no results found'}
                                        hp_bot.general.errorHandler(arg,adult,startdate,cityname,cityid,message,'Mobile Parser',true)
                                    }
                                    else
                                    {
                                        message = {'info':'no results found'}
                                        hp_bot.general.errorHandler(arg,adult,startdate,cityname,cityid,message,'Mobile Parser',true)
                                    }
                                }

                                counterror = 0;
                            }

                            //standart IP: 192.168.56.1
                            setTimeout(()=>{
                                hp_bot.general.getNewIP(ipischanging).then(gotNewIP=>
                                {
                                    counterror++
                                    console.log('IP changes counter : '+counterror)
                                    ipischanging=false;
                                    startFetch(adult);

                                })
                            },1000)
                            ipischanging =true;

                        }

                        console.log("PARSED RESULT "+ parsed)


                    }
                 startFetch(adult);

            })
        },
        "standart"    : async function startSeleniumJOB(arg)              {
            let isfetching = true;
            console.log("->Fetching Selenium Version");
            console.log("Selenium is still in testing mode.. ");

            //IF NO CITY ID go to NEXT JOB
            if(arg.cityid[0]=='' || !arg.cityid[0])
            {
                hp_bot.job.nextJob(startSeleniumJOB);
                return
            }

            console.log("JOB FOUND:");
            console.log(arg);
            console.log("=====================");
            currentip = await _CONTROLLER.showExternalIP(true)

            let city        = arg.name.replace(" ", '+');
            let icity       =  0;
            let countcity   =  arg.cityid.length;

            let idate       =  0;
            let countdate   =  arg.dates.length;


            nextCityId();
            function nextCityId()     {
                //Check if we have made all CityID
                if(icity==countcity && isfetching){
                    console.log("Job Done");
                    console.log(JSON.stringify(_HP.hotels.Booking));
                    hp_bot.job.nextJob(startSeleniumJOB);
                }
                //Start fetching next CityID
                if(icity<countcity)
                {
                    idate=0;
                    nextDate(arg.cityid[icity])
                    icity++;
                }

            }
            function nextDate(cityid) {
                //Clear after new date
                _HP.hotels.Booking = [];
                const start        = arg.dates[idate];
                fetchData(1,start,city,cityid);
                idate++;
            }
            function fetchData(adult,start,city,cityid) {

                const end  = _CONTROLLER.time(start,'YYYY-MM-DD').add(1, 'day').format('YYYY-MM-DD');
                let room  = 1;
                let child  = 1;

                _HP.methods.Booking.get(cityid,city,start,end,room,adult,child,false,arg.job,arg.id).then(result=>{

                    let isok = -1;

                    //go to next adult
                    if(isok==1||isok=='1'){
                        fetchData(2,start,city,cityid);
                    }

                    //got to next date
                    //send result to server
                    if(isok==2||isok=='2')
                    {
                        //send job
                        _HP.methods.general.generateJson(_HP.hotels.Booking,arg.job,arg.id,start,slave).then(answer=>{

                            //next cityid
                            if(idate==countdate){
                                hp_bot.job.updateOldJob(start,cityid,arg.dates);
                                nextCityId()
                            }

                            //next cityid
                            if(idate<countdate){
                                //Go to Nekt Date
                                console.log("=> going to next date: "+idate);
                                hp_bot.job.updateOldJob(start,null,null);
                                nextDate(cityid)
                            }

                        })
                    }

                    //Get new IP Address
                    if(isok==0||isok=='0'){
                        //standart IP: 192.168.56.1
                        hp_bot.general.getNewIP().then(gotNewIP=>{
                            fetchData(adult,start,city,cityid);
                        })
                    }

                })

            }

        },
        "api"         : async function startApiJob()                      {}
    },
    general:{
        "generateDeviceID":   function generateDeviceID(show = false)     {
            //

            let nr1 = Math.floor((Math.random() * 9999) + 1000);
            let nr2 = Math.floor((Math.random() * 9999) + 1000);
            let nr3 = Math.floor((Math.random() * 1) + 10);
            if(show)
                console.log('generateDeviceID: '+ `72331403-${nr1}-${nr2}-${nr3}`)

            return `72331403-${nr1}-${nr2}-${nr3}`
        },
        "getNewIP":           function getNewIP(ischanging=false)                         {

            return new Promise(resolve=>{

                console.log('old ip: ' + currentip);

                hp_bot.general.generateDeviceID(true);

                _CONTROLLER.tor.newTorSession(async function(err){
                    if(err)console.log(err)
                    currentip    =  await _CONTROLLER.showExternalIP(false);
                    console.log("new ip:"+currentip);
                    setTimeout(()=>{
                        resolve(currentip)
                    },5000)

                })

                /*
                if(ischanging)
                    socket.emit('newip',slave, async function(status) {
                        console.log("callback")
                        currentip    =  await _CONTROLLER.showExternalIP(false)

                        if(!currentip)
                            setTimeout(()=>{getNewIP();},1000);
                        else
                        {
                            console.log("new ip:"+currentip)
                            resolve(currentip)
                        }
                    });

                else{
                    setTimeout(()=>{
                        resolve(currentip)
                    },40000)
                }
                */
            })
        },
        "getXMLData":         function fetchIphoneXMLData(adult,start,end,city,cityid) {
            return new Promise(resolve=>{
                const devid = hp_bot.general.generateDeviceID();
                const _url    = `https://iphone-xml.booking.com/json/bookings.getHotelAvailabilityMobile?detail_level=1&genius_freebies=2&show_distance_cc=1&request_less_fields=2&show_facilities_review_score=1&show_last_minute_saving=1&include_translations=2&add_bh_info=1&languagecode=de&urgency=1&include_taxes=1&include_count=1&location_source=autocomplete&show_price_trend=1&show_deals=flash%2Clastm%2Csmart%2Cgenius&children_qty=0&low_av_alternatives=0&&&order_by=popularity&show_if_class_is_estimated=1&show_categories=v3&show_location_score=1&show_last_reservation_text=1&show_reinforcement_text=v2&show_filtered_facilities=1&show_if_no_cc_allowed=2&show_soldout=1&show_is_personalized_result=0&show_flash_saving=1&max_persons=-2&use_direct_payment=1&add_ctrip_info=1&include_rack_rate_savings=1&&include_url=1&bh_alt_urgency_msg=1&show_refundable=1&check_price_is_final=1&show_extra_charges=1&room_qty=1&show_if_city_center=1&available_rooms=0&price_buckets=100&flash_deals_amount_if_unsigned=2&show_low_availability=2&show_business_badge=1&autoextend=1&user_os=6.0.1&user_version=11.3.1-android&network_type=wifi&bt=1&arrival_date=${start}&departure_date=${end}&city_ids=${cityid.replace('city=', '').replace('|si=ai%2Cco%2Cci%2Cre%2Cdi', '')}&rows=5000&device_id=${devid}da-ef8edab1ad7f&guest_qty=${adult}`;
                const options =  {
                    headers: {
                        'Content-Type' : 'application/x-www-form-urlencoded',
                        'Authorization': 'Basic dGhlc2FpbnRzYnY6ZGdDVnlhcXZCeGdN'
                    },
                    url: _url,
                    timeout: 900000
                };

                //Send request
                _CONTROLLER.tor.request(options, function (err, resp, body) {
                    if(body)
                    resolve(JSON.parse(body));
                    else
                        console.log('no data')
                })
            })
        },
        "validateXmlResult":  function (xml,arg,adult,start,_city,cityid) {

            console.log("check validate");

            let xmlcity = null;

            //remove space and + at start and at the end
            const cityname = arg.name.trim().replace('/+/g','');
            const listcity = [];

            try
            {

                let found           = false;
                xmlcity         = xml.filter;

                if(xmlcity)
                    xmlcity.forEach(fitler=>
                    {
                        if(fitler.id == 'city')
                        {
                            if(fitler.categories)
                                fitler.categories.forEach(categorycity=>{
                                    listcity.push(categorycity.name);
                                    if(categorycity.name.toUpperCase().indexOf(cityname.toUpperCase())>=0)
                                        found = categorycity;
                                })
                        }
                    });

                if(found==false && listcity.length>1)
                {
                    let message = {'warrning':'city_ids did not match cityname '+cityname,
                        'list':listcity}

                    hp_bot.general.errorHandler(arg,adult,start,cityname,cityid,message,'Mobile Parser',true);
                    return null;
                }
                else
                {
                    console.log("Data is valid")
                    return true;
                }

            }
            catch (TypeError)
            {
                console.log(" error validate date")
                console.log(xmlcity)
                console.log(TypeError)
                return true;
            }

        },
        "errorHandler":       function (arg,adult,start,city,cityid,message,type='mobile',rebootanddelete=false) {

            console.log( "errorHandler " + type)
            city =city.replace('/+/g','');

            const hosturl='http://'+hostip+':'+hostport+'/errorjob';
            let log = {
                "job"    :   JSON.stringify(arg),
                "jobid"  :   arg.job,
                "message":   JSON.stringify(message) + ',{ "cityid":'+cityid+',"city":"'+city+'" }',
                "cityid" :   cityid,
                "city"   :   city,
                "slave"  :   slave,
                "client" :   client,
            };

            _CONTROLLER.request.post({
                    headers: {'content-type' : 'application/x-www-form-urlencoded'},
                    url:      hosturl,
                    json:     true,
                    form:     {
                        data: JSON.stringify(log)
                    }
                },
                function(error, response, body){

                    //Remove backup if error is not acceptable
                    //example if cityid is incorrect
                    if(rebootanddelete){
                 //       _CONTROLLER.exec('rm /var/node/backup.json');
                    }
                });
            setTimeout(()=>{
                if(rebootanddelete){}
               //     _CONTROLLER.reboot();
                else
                    _CONTROLLER.version();
            },10000)

        }
    },
};

//EXPORT Functions
hp_bot.constructor.prototype.start       = hp_bot.init;
hp_bot.constructor.prototype.getNewJob   = hp_bot.job.getNewJob;
module.exports                           = hp_bot.constructor;


//###Error Output
process.on('uncaughtException', function (error) {
    console.log(error.stack);
});

// Add a connect listener
socket.on('connect',            function (socket) {
    console.log('Connected!');
});
socket.on('disconnect',         function(){
    console.log("lost connection to server")
    socket.connect()
    socket.emit('slaves',slave)
    hp_bot.init();
    // reconnect
});