| 1 | {"version":3,"sources":["../../src/strategies/LinearProxyStrategy.js"],"names":["LinearProxyStrategy","strategy","maxPrimaryProxiesCount","primaryProxiesProbablity","primaryProxies","primaryProxiesCount","secondaryProxies","secondaryProxyIteration","kpdSum","update","proxies","forEach","proxy","stats","kpd","countKpd","probability","sortedProxies","p","isWorkingProxy","filter","slice","length","list","Err","initProbability","Math","random","sumProbability","i","success","fatal","error","count","avgTime","errorTime","ProxyStrategy"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AACA;;AACA;;AACA;;AACA;;AACA;;AAEA;;;;;;AACA;IAEaA,mB;;;;;;;;;;;;;;;UACXC,Q,GAAW,Q;UAKXC,sB,GAAyB,E;UACzBC,wB,GAA2B,I;UAG3BC,c,GAAiB,E;UACjBC,mB,GAAsB,C;UACtBC,gB,GAAmB,E;UACnBC,uB,GAA0B,C;UAC1BC,M,GAAS,C;;;;;;WAET;AACA,wBAAW;AACT,WAAKC,MAAL;AACA,aAAO;AACLJ,QAAAA,mBAAmB,EAAE,KAAKA,mBADrB;AAELE,QAAAA,uBAAuB,EAAE,KAAKA,uBAFzB;AAGLC,QAAAA,MAAM,EAAE,KAAKA;AAHR,OAAP;AAKD;;;WACD,kBAAS;AAAA;;AACP,UAAME,OAAO,GAAG,sBAAI,IAAJ,EAAU,oBAAV,CAAhB;AACA,UAAI,CAACA,OAAL,EAAc;AACdA,MAAAA,OAAO,CAACC,OAAR,CAAgB,UAACC,KAAD,EAAW;AACzBA,QAAAA,KAAK,CAACC,KAAN,CAAYC,GAAZ,GAAkB,MAAI,CAACC,QAAL,CAAcH,KAAd,CAAlB;AACAA,QAAAA,KAAK,CAACC,KAAN,CAAYG,WAAZ,GAA0B,CAA1B;AACD,OAHD;AAIA,UAAMC,aAAa,GAAG,wBAAOP,OAAP,EAAgB,UAACQ,CAAD;AAAA,eAAO,CAAC,sBAAIA,CAAJ,EAAO,WAAP,CAAR;AAAA,OAAhB,CAAtB;;AACA,UAAMC,cAAc,GAAG,SAAjBA,cAAiB,CAACD,CAAD;AAAA,eAAO,sBAAIA,CAAJ,EAAO,WAAP,EAAoB,CAApB,IAAyB,CAAzB,IAA8B,sBAAIA,CAAJ,EAAO,aAAP,EAAsB,CAAtB,IAA2B,CAAhE;AAAA,OAAvB,CARO,CAUP;;;AACA,WAAKd,cAAL,GAAsBa,aAAa,CAACG,MAAd,CAAqBD,cAArB,EAAqCE,KAArC,CAA2C,CAA3C,EAA8C,KAAKnB,sBAAnD,CAAtB;AACA,WAAKG,mBAAL,GAA2B,KAAKD,cAAL,CAAoBkB,MAA/C;AACA,WAAKhB,gBAAL,GAAwB,yBAAQW,aAAa,CAACI,KAAd,CAAoB,KAAKhB,mBAAzB,CAAR,CAAxB,CAbO,CAeP;;AACA,WAAKG,MAAL,GAAc,uBAAM,KAAKJ,cAAX,EAA2B,UAACc,CAAD;AAAA,eAAO,sBAAIA,CAAJ,EAAO,WAAP,CAAP;AAAA,OAA3B,CAAd;AACA,UAAI,KAAKV,MAAL,GAAc,CAAlB,EAAqB,KAAKA,MAAL,GAAc,CAAd,CAjBd,CAiB+B;;AAEtC,WAAKJ,cAAL,CAAoBO,OAApB,CAA4B,UAACC,KAAD,EAAW;AACrC,YAAI,MAAI,CAACJ,MAAT,EAAiB;AACf;AACAI,UAAAA,KAAK,CAACI,WAAN,GAAqB,sBAAIJ,KAAJ,EAAW,WAAX,IAA0B,MAAI,CAACJ,MAAhC,GAA0C,MAAI,CAACL,wBAAnE,CAFe,CAE8E;AAC9F,SAHD,MAGO;AACL;AACAS,UAAAA,KAAK,CAACI,WAAN,GAAqB,IAAI,MAAI,CAACX,mBAAV,GAAiC,MAAI,CAACF,wBAA1D;AACD;AACF,OARD;AASD;;;WAED,oBAAW;AACT,UAAMoB,IAAI,GAAG,sBAAI,IAAJ,EAAU,oBAAV,CAAb;AACA,UAAI,EAACA,IAAD,aAACA,IAAD,eAACA,IAAI,CAAED,MAAP,CAAJ,EAAmB,MAAM,IAAIE,eAAJ,CAAQ,0BAAR,CAAN;AAEnB,UAAMC,eAAe,GAAGC,IAAI,CAACC,MAAL,EAAxB;AACA,UAAIC,cAAc,GAAG,CAArB;;AACA,WAAK,IAAIC,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG,KAAKzB,cAAL,CAAoBkB,MAAxC,EAAgDO,CAAC,IAAI,CAArD,EAAwD;AACtDD,QAAAA,cAAc,IAAI,CAAC,sBAAI,KAAKxB,cAAT,YAA4ByB,CAA5B,wBAAD,IAAuD,CAAzE;AACA,YAAIJ,eAAe,GAAGG,cAAtB,EAAsC,OAAO,KAAKxB,cAAL,CAAoByB,CAApB,CAAP;AACvC;;AAED,WAAKtB,uBAAL,IAAgC,CAAhC;AACA,aAAO,KAAKD,gBAAL,CAAsB,KAAKC,uBAAL,GAA+B,KAAKD,gBAAL,CAAsBgB,MAA3E,CAAP;AACD,K,CAED;;;;WACA,kBAASV,KAAT,EAAgB;AACd,UAAMkB,OAAO,GAAG,sBAAIlB,KAAJ,EAAW,wBAAX,EAAqC,CAArC,CAAhB;AACA,UAAMmB,KAAK,GAAG,sBAAInB,KAAJ,EAAW,aAAX,EAA0B,CAA1B,CAAd;AACA,UAAImB,KAAJ,EAAW,OAAO,CAAE,CAAT;AACX,UAAMC,KAAK,GAAG,sBAAIpB,KAAJ,EAAW,sBAAX,EAAmC,CAAnC,CAAd;AACA,UAAMqB,KAAK,GAAG,sBAAIrB,KAAJ,EAAW,OAAX,EAAoB,CAApB,CAAd;AACA,UAAI,CAACqB,KAAL,EAAY,OAAO,CAAP,CANE,CAMQ;;AACtB,UAAMC,OAAO,GAAGD,KAAK,GAAG,sBAAIrB,KAAJ,EAAW,MAAX,EAAmB,CAAnB,IAAwB,sBAAIA,KAAJ,EAAW,OAAX,EAAoB,CAApB,CAA3B,GAAoD,CAAzE;AACA,UAAMuB,SAAS,GAAGH,KAAK,GAAG,sBAAIpB,KAAJ,EAAW,oBAAX,EAAiC,CAAjC,IAAsCoB,KAAzC,GAAiD,CAAxE;AACA,aAAO,SAASE,OAAO,GAAIF,KAAK,GAAGF,OAAT,GAAoBK,SAAvC,CAAP;AACD;;;;+FAED;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA,uBAEQ,KAAK1B,MAAL,EAFR;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,O;;;;;;;;;;EAnFuC2B,6B;;;eAyF1BpC,mB","sourcesContent":["/* eslint-disable no-param-reassign */\nimport Err from '@lskjs/err';\nimport get from 'lodash/get';\nimport shuffle from 'lodash/shuffle';\nimport sortBy from 'lodash/sortBy';\nimport sumBy from 'lodash/sumBy';\n\nimport { ProxyStrategy } from './ProxyStrategy';\n// import request from './request';\n\nexport class LinearProxyStrategy extends ProxyStrategy {\n strategy = 'linear';\n // consts\n // 10 проксей парсят\n // на них выделяется 95% попыток парсинга\n // на остальные прокси выделяется 5% попыток, потому что, вдруг они там за это время починились\n maxPrimaryProxiesCount = 10;\n primaryProxiesProbablity = 0.95;\n\n // vars\n primaryProxies = [];\n primaryProxiesCount = 0;\n secondaryProxies = [];\n secondaryProxyIteration = 0;\n kpdSum = 0;\n\n //\n getStats() {\n this.update();\n return {\n primaryProxiesCount: this.primaryProxiesCount,\n secondaryProxyIteration: this.secondaryProxyIteration,\n kpdSum: this.kpdSum,\n };\n }\n update() {\n const proxies = get(this, 'manager.cache.list');\n if (!proxies) return;\n proxies.forEach((proxy) => {\n proxy.stats.kpd = this.countKpd(proxy);\n proxy.stats.probability = 0;\n });\n const sortedProxies = sortBy(proxies, (p) => -get(p, 'stats.kpd'));\n const isWorkingProxy = (p) => get(p, 'stats.kpd', 0) > 0 && get(p, 'stats.count', 0) > 0;\n\n // количество реальных прокси\n this.primaryProxies = sortedProxies.filter(isWorkingProxy).slice(0, this.maxPrimaryProxiesCount);\n this.primaryProxiesCount = this.primaryProxies.length;\n this.secondaryProxies = shuffle(sortedProxies.slice(this.primaryProxiesCount));\n\n // тот КПД который будет разыгрываться между основными проксями\n this.kpdSum = sumBy(this.primaryProxies, (p) => get(p, 'stats.kpd'));\n if (this.kpdSum < 1) this.kpdSum = 0; // если суммарного кпд еще мало, то считаем что его нет (ну мелочь всякая пока, не попарсили)\n\n this.primaryProxies.forEach((proxy) => {\n if (this.kpdSum) {\n // если есть суммарный кпд то разыгрываеи его\n proxy.probability = (get(proxy, 'stats.kpd') / this.kpdSum) * this.primaryProxiesProbablity; // TODO, как гипотезу можно рассмотреть какую нибудь не линейную функцию\n } else {\n // если его нет, то распределяем вероятность между проксями линейно\n proxy.probability = (1 / this.primaryProxiesCount) * this.primaryProxiesProbablity;\n }\n });\n }\n\n getProxy() {\n const list = get(this, 'manager.cache.list');\n if (!list?.length) throw new Err('PROXY_MANAGER_LIST_EMPTY');\n\n const initProbability = Math.random();\n let sumProbability = 0;\n for (let i = 0; i < this.primaryProxies.length; i += 1) {\n sumProbability += +get(this.primaryProxies, `${i}.stats.probability`) || 0;\n if (initProbability < sumProbability) return this.primaryProxies[i];\n }\n\n this.secondaryProxyIteration += 1;\n return this.secondaryProxies[this.secondaryProxyIteration % this.secondaryProxies.length];\n }\n\n // эстимация количества спаршенных видео в минуту\n countKpd(proxy) {\n const success = get(proxy, 'stats.statuses.success', 0);\n const fatal = get(proxy, 'stats.fatal', 0);\n if (fatal) return - 1;\n const error = get(proxy, 'stats.statuses.error', 0);\n const count = get(proxy, 'count', 0);\n if (!count) return 0; // 1 - один по умолчанию потому что они должны быть в списке круче чем сломанные прокси у которых 0\n const avgTime = count ? get(proxy, 'time', 0) / get(proxy, 'count', 1) : 0;\n const errorTime = error ? get(proxy, 'statusesTime.error', 0) / error : 0;\n return 60000 / (avgTime + (error / success) * errorTime);\n }\n\n async run() {\n await super.run();\n await this.update();\n }\n}\n\nexport default LinearProxyStrategy;\n"],"file":"LinearProxyStrategy.js"} |