This week only. Pastebin PRO Accounts Christmas Special! Don't miss out!Want more features on Pastebin? Sign Up, it's FREE!
Guest

Untitled

By: a guest on May 17th, 2014  |  syntax: Python  |  size: 2.83 KB  |  views: 217  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
This paste has a previous version, view the difference. Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. # -*- coding: utf-8 -*-
  2.  
  3. import urllib.request
  4. import json
  5. import operator
  6. import os
  7. from collections import defaultdict
  8. from multiprocessing.dummy import Pool
  9. import threading
  10.  
  11. class Counter:
  12.     def __init__(self, i=0):
  13.         self.i = i
  14.         # create a lock
  15.         self.lock = threading.Lock()
  16.  
  17.     def __iter__(self):
  18.         return self
  19.  
  20.     def next(self):
  21.         # acquire/release the lock when updating self.i
  22.         with self.lock:
  23.             self.i += 1
  24.             return self.i
  25.    
  26.     def current(self):
  27.         return self.i
  28.  
  29. os.system('cls')
  30.  
  31. worker_pool_size = 20
  32. worker_pool = Pool(worker_pool_size)
  33.  
  34. games = 5000
  35. game_range = [860000, 970000]
  36.  
  37. counter = Counter()
  38.  
  39. chunk_size = 3
  40.  
  41. common_words = defaultdict(int)
  42. common_champions = defaultdict(int)
  43.  
  44. exclude = ['a', 'i', 'me', 'why', 'to', 'and', 'my', 'you', 'dont', 'with', 'what', 'in',
  45. 'are', 'more', 'so', 'have', 'not', 'for', 'no', 'is', 'yes', 'im', 'u', '?', 'the', 'or',
  46. 'do', 'this', 'your', 'on', 'we', 'they', 'our', 'it', 'just', 'go', 'he', 'all', 'that',
  47. 'ur', 'who', 'of', 'can', 'its', 'get', 'cant', 'it\'s', 'can\'t', 'don\'t', 'i\'m', 'now',
  48. 'if', 'got', 'ss']
  49.  
  50.  
  51. def loading_logic(i):
  52.     if counter.current() >= games:
  53.         return
  54.    
  55.     try:
  56.         request = urllib.request.urlopen('http://euw.leagueoflegends.com/tribunal/en/get_reform_game/' + str(i) + '/1/')
  57.     except:
  58.         return
  59.            
  60.     data = request.read()
  61.     data = data.decode('utf-8')
  62.    
  63.     if (data == ''):
  64.         return
  65.    
  66.     data = json.loads(data)
  67.    
  68.     for message in data['chat_log']:
  69.         if message['association_to_offender'] == 'offender':
  70.             words = message['message'].split()
  71.            
  72.             for word in words:
  73.                 word = word.lower()
  74.                 if not word.isdigit() and word not in exclude:
  75.                     common_words[word] += 1
  76.    
  77.     for player in data['players']:
  78.         if player['association_to_offender'] == 'offender':
  79.             common_champions[player['champion_name']] += 1
  80.             break
  81.    
  82.     counter.next()
  83.  
  84.  
  85. for i in range(game_range[0], game_range[1], worker_pool_size*chunk_size):
  86.     os.system('cls')
  87.     print('%s/%s games analyzed' % (counter.current(), games))
  88.     worker_pool.map(loading_logic, range(i, i+worker_pool_size*chunk_size))
  89.     if counter.current() >= games:
  90.         break
  91.  
  92.  
  93. common_words = sorted(common_words.items(), key=operator.itemgetter(1), reverse=True)
  94. common_champions = sorted(common_champions.items(), key=operator.itemgetter(1), reverse=True)
  95.  
  96. print('Words used: ', len(common_words))
  97.  
  98. for i in range(30):
  99.     print(common_words[i][0], ' - ', common_words[i][1])
  100.        
  101. print()
  102. print()
  103.  
  104. for i in range(30):
  105.         print(common_champions[i][0], ' - ', common_champions[i][1])
clone this paste RAW Paste Data