API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Oct 16th, 2018
68
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 21.57 KB | None | 0 0
raw download clone embed print report
  1. const Mika = require('mika');
  2. const util = require('util');
  3. const squel = require('squel');
  4. const fs = require('fs');
  5. const Bottleneck = require('bottleneck');
  6. const Moment = require('moment');
  7. const MomentRange = require('moment-range');
  8. const mika = new Mika();
  9. const moment = MomentRange.extendMoment(Moment);
  10. const shared = new Int32Array(new SharedArrayBuffer(4));
  11.  
  12. class MatchesRetriever {
  13.  
  14. constructor() {
  15. this.matchesTypes = {
  16. all: 'all',
  17. pro: 'pro',
  18. players: 'players'
  19. };
  20. this.matches_ids = [];
  21. this.heroes_players = {};
  22. this.players_matches = {};
  23. this.heroes = [];
  24. this.iterator = 0;
  25. this.timeoutOnError = 30;
  26.  
  27. this.limiter = new Bottleneck({
  28. maxConcurrent: 60,
  29. minTime: 500,
  30. strategy: Bottleneck.strategy.BLOCK,
  31. reservoir: null
  32. });
  33. }
  34.  
  35. static unique(arr) {
  36. return [...new Set(arr)];
  37. }
  38.  
  39. static msleep(n) {
  40. Atomics.wait(shared, 0, 0, n);
  41. }
  42.  
  43. static sleep(n) {
  44. MatchesRetriever.msleep(n * 1000);
  45. }
  46.  
  47. removeEmpty(obj, filteredValues = []) {
  48. Object.keys(obj).forEach((k) => ((!obj[k] || (obj[k] && filteredValues.includes(obj[k]))) && obj[k] !== undefined) && delete obj[k]);
  49. return obj;
  50. };
  51.  
  52. static _getQueryForProMatches() {
  53. return squel.select({separator: '\n'})
  54. .field('matches.match_id')
  55. .field('matches.start_time')
  56. .field('((player_matches.player_slot < 128) = matches.radiant_win) win')
  57. .field('player_matches.hero_id')
  58. .field('player_matches.account_id')
  59. .field('leagues.name leaguename')
  60. .from('matches')
  61. .join('match_patch using(match_id)')
  62. .join('leagues using(leagueid)')
  63. .join('player_matches using(match_id)')
  64. .join('heroes on heroes.id = player_matches.hero_id')
  65. .left_join('notable_players', null, 'notable_players.account_id = player_matches.account_id AND notable_players.locked_until = ' +
  66. '(' +
  67. squel.select()
  68. .from('notable_players')
  69. .field('MAX(locked_until)')
  70. .toString()
  71. + ')'
  72. )
  73. .left_join('teams using(team_id)')
  74. .where('TRUE')
  75. .order('matches.match_id DESC NULLS LAST', null);
  76. }
  77.  
  78. getTopPlayersForAllHeroes(playersLimit = 10, from = null) {
  79. let that = this;
  80. let date = from ? moment(from, "DD.MM.YYYY") : moment();
  81. let days = date.diff(moment(), 'days');
  82.  
  83. that.matches_ids = [];
  84.  
  85. mika.getHeroes()
  86. .then((data) => {
  87. if (data !== null) {
  88. that.heroes = {...data};
  89. that._getTopPlayersForAllHeroes(that.heroes, playersLimit);
  90.  
  91. that.limiter.once('empty', function () {
  92. util.log('Start to get all players matches');
  93. that.limiter.schedule(() => Promise.resolve()
  94. .then(() => {
  95. that._getAllHeroesTopPlayersMatches(that.heroes_players, days);
  96.  
  97. that.limiter.once('empty', function () {
  98. util.log('Retrieving of players matches has been finished');
  99. that.limiter.schedule(() => Promise.resolve()
  100. .then(() => {
  101. that.heroes_players.total_matches = that.matches_ids.length;
  102. util.log('total matches: ', that.heroes_players.total_matches);
  103. that.matches_ids = MatchesRetriever.unique(that.matches_ids);
  104. that.heroes_players.unique_matches = that.matches_ids.length;
  105. util.log('unique matches: ', that.heroes_players.unique_matches);
  106. fs.writeFile('./heroes_players_matches.json', JSON.stringify(that.heroes_players), (err) => {
  107. if (err) throw err;
  108. });
  109.  
  110. fs.writeFile('./matches.json', JSON.stringify(that.matches_ids), (err) => {
  111. if (err) throw err;
  112. });
  113. util.log('All matches has been saved');
  114. }));
  115. });
  116. }));
  117. });
  118. }
  119. }).catch((err) => {
  120. util.log('getHeroes: ', err);
  121. });
  122. }
  123.  
  124. _getTopPlayersForAllHeroes(data, playersLimit) {
  125. let that = this;
  126. that.limiter.updateSettings({reservoir: null});
  127.  
  128. Object.keys(data).forEach(function (key) {
  129. let hero = that.heroes[key];
  130. that.limiter.schedule(hero => mika.getRankings(hero.id), hero)
  131. .then((players) => {
  132. that.iterator = 0;
  133. that.heroes_players[hero.id] = {};
  134. util.log('heroId: ', hero.id);
  135. players.rankings.every(function (player) {
  136. that.heroes_players[hero.id][player.account_id] = {
  137. 'id': player.account_id,
  138. 'score': player.score,
  139. 'rank': player.rank_tier
  140. };
  141. that.iterator++;
  142. if (that.iterator < playersLimit) {
  143. return true;
  144. }
  145. });
  146. })
  147. .catch(err => {
  148. util.log('getTopPlayers: ', err, 'id=' + hero.id);
  149. that.limiter.updateSettings({reservoir: 0});
  150. util.log('sleep for ' + (that.timeoutOnError) + ' sec');
  151. MatchesRetriever.sleep(that.timeoutOnError);
  152. let new_data = {
  153. [key]: hero
  154. };
  155. that._getTopPlayersForAllHeroes(new_data, playersLimit);
  156.  
  157. });
  158. });
  159. };
  160.  
  161. _getPlayersMatches(player, days, game_modes) {
  162. let that = this;
  163. that.limiter.updateSettings({reservoir: null});
  164. let date = moment();
  165. that.players_matches[player] = that.players_matches[player] || {
  166. game_modes: game_modes,
  167. player_id: player,
  168. days: days,
  169. date: date.format('DD.MM.YYYY'),
  170. matches: {}
  171. };
  172.  
  173. game_modes.forEach((game_mode) => {
  174. that.limiter.schedule((player) => mika.getPlayerMatches(player, that.removeEmpty({
  175. date: days,
  176. game_mode: game_mode
  177. }, ['all'])), player)
  178. .then((matches) => {
  179. that.iterator += matches.length;
  180. util.log('playerMatches: ', matches.length, player);
  181. that.players_matches[player].matches[game_mode] = matches;
  182. that.players_matches[player].matches.total_matches = matches.length;
  183. if (game_mode === game_modes[game_modes.length - 1]) {
  184. fs.writeFile('./players/matches_' + player + '.json', JSON.stringify(that.players_matches[player]), (err) => {
  185. delete that.players_matches[player];
  186. if (err) throw err;
  187. });
  188. }
  189. })
  190. .catch(err => {
  191. util.log('getPlayerMatches: ', err, 'player: ', player);
  192. that.limiter.updateSettings({reservoir: 0});
  193. util.log('sleep for ' + (that.timeoutOnError) + ' sec');
  194. MatchesRetriever.sleep(that.timeoutOnError);
  195. that._getPlayersMatches(player, days, game_modes);
  196. });
  197. })
  198. }
  199.  
  200. getPlayersMatches(players, days, game_modes = ['all']) {
  201. let that = this;
  202. that.iterator = 0;
  203.  
  204. let run = (player, days, game_modes) => {
  205. players.forEach((player) => {
  206. that._getPlayersMatches(player, days, game_modes);
  207. });
  208.  
  209. that.limiter.once('empty', () => {
  210. util.log('Retrieving of player\'s matches has been finished');
  211. that.limiter.schedule(() => Promise.resolve()
  212. .then(() => {
  213. /* Object.keys(that.players_matches).forEach((player) => {
  214. fs.writeFile('./players/matches_' + player + '.json', JSON.stringify(that.players_matches[player]), (err) => {
  215. if (err) throw err;
  216. });
  217. });*/
  218. util.log('total matches: ', that.iterator);
  219. util.log('All matches has been saved');
  220. }));
  221. });
  222. };
  223.  
  224. if (!Array.isArray(players)) {
  225. fs.readFile('./players/players.json', (err, data) => {
  226. if (data) {
  227. players = JSON.parse(data);
  228. run(players, days, game_modes)
  229. }
  230. });
  231. }
  232. else {
  233. run(players, days, game_modes)
  234. }
  235. }
  236.  
  237. _getAllHeroesTopPlayersMatches(heroes, days) {
  238. let that = this;
  239. that.iterator = 0;
  240. that.limiter.updateSettings({reservoir: null});
  241.  
  242. Object.keys(heroes).forEach(function (key) {
  243. let hero = heroes[key];
  244. Object.keys(hero).forEach(function (index) {
  245. let player = hero[index];
  246. that.limiter.schedule(player => mika.getPlayerMatches(player, {date: days}), player.id)
  247. .then((matches) => {
  248. that.iterator++;
  249. util.log('playerMatches: ', that.iterator, player.id);
  250. that.heroes_players[key][index].matches = [];
  251. matches.forEach(function (match) {
  252. that.heroes_players[key][index].matches.push(match.match_id);
  253. that.matches_ids.push(match.match_id);
  254. });
  255. })
  256. .catch(err => {
  257. util.log('getPlayerMatches: ', err, 'hero: ', key, 'player: ', player.id);
  258. that.limiter.updateSettings({reservoir: 0});
  259. util.log('sleep for ' + (that.timeoutOnError) + ' sec');
  260. MatchesRetriever.sleep(that.timeoutOnError);
  261. let heroes_new = {
  262. [key]: {
  263. [index]: player
  264. }
  265. };
  266. that._getAllHeroesTopPlayersMatches(heroes_new, days);
  267. });
  268. });
  269. });
  270. };
  271.  
  272. getProLeagueMatches(from = null, limit = 100) {
  273. let that = this;
  274. limit *= 10;
  275. fs.readFile('./matches_pro.json', (err, data) => {
  276. if (!data) {
  277. that.matches_ids = [];
  278. }
  279. else {
  280. that.matches_ids = JSON.parse(data);
  281. }
  282.  
  283. let date = from ? moment(from, 'DD.MM.YYYY') : moment();
  284. let range = moment().range(date, moment());
  285. let array = Array.from(range.by('months'));
  286. array.push(moment().add(1, 'day'));
  287. let query = MatchesRetriever._getQueryForProMatches();
  288. array.reverse().some((date, index, array) => {
  289. if (!Object.is(array.length - 1, index)) {
  290. let newQuery = query.clone();
  291. newQuery = newQuery
  292. .where('matches.start_time >= extract(epoch from timestamp \'' + array[index + 1].format('MM-DD-YYYY') + 'T00:00:00.000Z\')')
  293. .where('matches.start_time < extract(epoch from timestamp \'' + date.format('MM-DD-YYYY') + 'T00:00:00.000Z\')')
  294. .toString();
  295.  
  296. that.limiter.schedule((query) => mika.explorer(query, true), newQuery)
  297. .then((data) => {
  298. util.log('start job');
  299. if (data !== null) {
  300. let lastIterations = false;
  301. if (limit > 0) {
  302. let uniqueMatches = data.rows.length;
  303. if (limit < uniqueMatches) {
  304. lastIterations = true;
  305. }
  306. else {
  307. limit -= data.rows.length;
  308. }
  309. }
  310. data.rows.some(match => {
  311. that.matches_ids.push(match.match_id);
  312. if (lastIterations) {
  313. limit--;
  314. if (limit <= 0) {
  315. util.log('limit');
  316. return true;
  317. }
  318. }
  319. });
  320. that.matches_ids = MatchesRetriever.unique(that.matches_ids);
  321. util.log('matches: ', that.matches_ids.length);
  322. fs.writeFile('./matches_pro.json', JSON.stringify(that.matches_ids), (err) => {
  323. if (err) throw err;
  324. if (limit <= 0) {
  325. that.limiter.stop({dropErrorMessage: 'Matches limit'})
  326. .then(() => {
  327. util.log('Shutdown completed!');
  328. });
  329. }
  330. util.log('pro matches ids saved');
  331. });
  332.  
  333. }
  334. }
  335. )
  336. .catch(function (err) {
  337. util.log(err);
  338. });
  339. }
  340. });
  341. });
  342.  
  343. }
  344.  
  345. downloadMatches(types = 'all', offset = 0, limit = null) {
  346. let that = this;
  347. that.iterator = 0;
  348.  
  349. switch (types) {
  350. case that.matchesTypes.players:
  351. let playersMatches = [];
  352. util.log('trying to retrieve matches');
  353. fs.readdir('./players', function (err, files) {
  354. playersMatches = files.filter(el => /^matches_.*\.json$/.test(el));
  355. let matches = null;
  356. let index = 0;
  357. const iMax = playersMatches.length;
  358. for (; index < iMax; index++) {
  359. matches = playersMatches[index];
  360. try {
  361. let data = fs.readFileSync('./players/' + matches);
  362. that.players_matches = JSON.parse(data);
  363. that.players_matches.game_modes.forEach((game_mode) => {
  364. that.players_matches.matches[game_mode].forEach((match) => {
  365. that.matches_ids.push(match.match_id);
  366. })
  367. });
  368. }
  369. catch (err) {
  370. throw err;
  371. }
  372. }
  373. that.matches_ids = MatchesRetriever.unique(that.matches_ids);
  374. util.log('total unique:', that.matches_ids.length, 'matches');
  375. that._getMatchesData(that.matches_ids, 'E:\\pro_pub_matches\\');
  376. });
  377. break;
  378. case that.matchesTypes.pro:
  379. fs.readFile('./matches_pro.json', function (err, data) {
  380. that.matches_ids = JSON.parse(data);
  381. that.matches_ids = MatchesRetriever.unique(that.matches_ids);
  382. if (offset > 0 || limit !== null) {
  383. that.matches_ids = that.matches_ids.slice(offset, limit);
  384. }
  385. util.log('total unique:', that.matches_ids.length, 'matches');
  386. that._getMatchesData(that.matches_ids, './matches/');
  387. });
  388. break;
  389. case that.matchesTypes.all:
  390. default:
  391. fs.readFile('./matches.json', function (err, data) {
  392. that.matches_ids = JSON.parse(data);
  393. that.matches_ids = MatchesRetriever.unique(that.matches_ids);
  394. if (offset > 0 || limit !== null) {
  395. that.matches_ids = that.matches_ids.slice(offset, limit);
  396. }
  397. util.log('total unique:', that.matches_ids.length, 'matches');
  398. /*fs.writeFile('./unique_matches.json', JSON.stringify(that.matches_ids), (err) => {
  399. if (err) throw err;
  400. });*/
  401. that._getMatchesData(that.matches_ids, './matches/');
  402. });
  403. }
  404. }
  405.  
  406. _getMatchesData(matches, path, parse = true) {
  407. let that = this;
  408. that.limiter.updateSettings({reservoir: null});
  409. try {
  410.  
  411. let index = 0;
  412. let percentage = 0;
  413. if(parse){
  414. !fs.existsSync(path) && fs.mkdirSync(path);
  415.  
  416. util.log('reading matches folder');
  417.  
  418. let files = fs.readdirSync(path);
  419. let file = '';
  420. let matchIndex;
  421. let iterator = 0;
  422. const iMax = files.length;
  423. for (; index < iMax; index++) {
  424. file = files[index];
  425. if(/\.json$/.test(file)){
  426. file = parseInt(file.replace('.json',''));
  427. matchIndex = matches.indexOf(file);
  428. if (matchIndex > -1){
  429. iterator++;
  430. delete matches[matchIndex];
  431. }
  432. }
  433. }
  434.  
  435. let filtered = matches.filter(function (el) {
  436. return el != null;
  437. });
  438. matches = filtered;
  439. util.log('filtered matches:', iterator);
  440. util.log('total unique matches after filtering of downloaded files:', matches.length);
  441. util.log('receiving matches in loop');
  442. }
  443.  
  444. let match = {};
  445. index = 0;
  446. percentage = 0;
  447. const iMax2 = matches.length;
  448. const percent = Math.floor(iMax2 / 100);
  449. (async function loop() {
  450. for (; index < iMax2; index++) {
  451. match = matches[index];
  452. if (index % percent === 0) {
  453. percentage++;
  454. util.log('past:', percentage + '%', 'index:', '#' + index, 'last match_id:', match);
  455. }
  456. if (match !== undefined && typeof match !== 'undefined') {
  457. await that.limiter.schedule((id) => mika.getMatch(id), match)
  458. .then((match) => {
  459. fs.writeFile(path + match.match_id + '.json', JSON.stringify(match), (err) => {
  460. if (err) throw err;
  461. util.log('downloaded:', '#' + index, 'match_id:', match.match_id);
  462. });
  463. })
  464. .catch(err => {
  465. util.log('getMatches: ', err, 'match: ', match, 'index: ', index);
  466. that.limiter.updateSettings({reservoir: 0});
  467. util.log('sleep for ' + (that.timeoutOnError) + ' sec');
  468. MatchesRetriever.sleep(that.timeoutOnError);
  469. let matches_new = [match];
  470. that._getMatchesData(matches_new, path, false);
  471. });
  472. }
  473. }
  474. })();
  475. }
  476. catch (err) {
  477. util.log(err);
  478. throw err;
  479. }
  480. }
  481. }
  482.  
  483. module.exports = MatchesRetriever;
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!