Guest User

Untitled

a guest
Apr 3rd, 2021
332
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 6.79 KB | None | 0 0
  1. import requests
  2. import concurrent.futures
  3. from Items import itemName
  4. from Realms import realmNames
  5. from Pets import petName
  6. import time
  7. import os
  8. from datetime import datetime
  9. import pickle
  10. import json
  11.  
  12. # how to install:
  13. # If you don't have a file for any of the imports above this, you need to install it through cmd.
  14. # If you try to run the program and something is missing or it doesn't run, install what's missing and try again.
  15. # Press your windows key + r, type "cmd" without quotes, type in the following:
  16. # pip install python
  17. # pip install pickle
  18. # pip install time
  19.  
  20.  
  21. # To run the program, install any IDLE that supports Python, I recommend Pycharm: https://www.jetbrains.com/pycharm/
  22. # Make sure all the associated files are located in the same file as your main file or else it won't work.
  23.  
  24. # change this if your PC can't handle 83 threads:
  25. MAX_THREADS = 83
  26.  
  27. itemDict = {}
  28. petDict = {}
  29. # This is used for your dictionaries:
  30. for block in itemName:
  31.     for item in itemName[block]:
  32.         itemDict[item] = {'price': block, 'name': itemName[block][item]}
  33.  
  34. for block in petName:
  35.     for item in petName[block]:
  36.         petDict[item] = {'price': block, 'name': petName[block][item]}
  37.  
  38.  
  39. # This is the multithreading section, it controls the download of urls and the search function:
  40. def download_url(url):
  41.     global realm_dict
  42.  
  43.     if int(url.split("?")[0].split("/")[-2]) in realmNames:
  44.         realm_name = realmNames[int(url.split("?")[0].split("/")[-2])]
  45.     else:
  46.         realm_name = "unknown"
  47.     try:
  48.         realmid = int(url.split("?")[0].split("/")[-2])
  49.         if realmid in realm_dict:
  50.             resp = requests.get(url, headers={'If-Modified-Since': realm_dict[realmid][0]})
  51.         else:
  52.             resp = requests.get(url)
  53.         if resp.status_code == 304:
  54.             pass  # print("not modified")
  55.         else:
  56.             data = {}
  57.             res = requests.get(url)
  58.             auctions = res.json()['auctions']
  59.  
  60.             realm_dict[realmid] = [res.headers['Last-Modified'],
  61.                                    datetime.strptime(res.headers['Last-Modified'][4:-4].strip(), '%d %b %Y %H:%M:%S')]
  62.  
  63.             with open("realm_timestamps.pickle", "wb") as f:
  64.                 pickle.dump(realm_dict, f)
  65.  
  66.             for i in auctions:
  67.                 if 'unit_price' in i:
  68.                     price = i['unit_price']
  69.                 elif 'buyout' in i:
  70.                     price = i['buyout']
  71.                 else:
  72.                     price = 0  # should never happen
  73.  
  74.                 item_id = i["item"]['id']
  75.  
  76.                 if price > 0:
  77.                     if item_id in itemDict:
  78.                         if price <= (itemDict[item_id]['price'] * 10000):
  79.                             try:
  80.                                 print(f"- ITEM - {realm_name}: {itemDict[item_id]['name']} | {price / 10000}g")
  81.                             except Exception as msg:
  82.                                 print(str(msg))
  83.                     elif 'pet_species_id' in i['item']:
  84.                         item_id = i['item']['pet_species_id']
  85.                         if item_id in petDict:
  86.                             if price <= (petDict[item_id]['price'] * 10000):
  87.                                 try:
  88.                                     if int(url.split("?")[0].split("/")[-2]) in realmNames:
  89.                                         print(f"- ITEM - {realm_name}: {petDict[item_id]['name']} | {price / 10000}g")
  90.                                     else:
  91.                                         print(f"- ITEM - Unknown Realm {petDict[item_id]['name']} | {price / 10000}g")
  92.                                 except Exception as msg:
  93.                                     print("pet id missing -----(" + str(msg) + ")" + str(item_id) + "<<")
  94.     except Exception as msg:
  95.         print(str(msg))
  96.         return data
  97.  
  98.  
  99. # Thread controller:
  100. def download_realm_data(urls):
  101.     threads = min(MAX_THREADS, len(urls))
  102.  
  103.     with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
  104.         executor.map(download_url, urls)
  105.  
  106.  
  107. # End multithreading functions
  108.  
  109.  
  110. # This creates your access token:
  111. def create_access_token(client_id, client_secret, region='us'):
  112.     data = {'grant_type': 'client_credentials'}
  113.     response = requests.post('https://%s.battle.net/oauth/token' % region, data=data, auth=(client_id, client_secret))
  114.     return response.json()
  115.  
  116.  
  117. # ends here
  118.  
  119. # This grabs urls for downloading:
  120. def get_realm_urls(token):
  121.     url = "https://us.api.blizzard.com/data/wow/connected-realm/index?namespace=dynamic-us&locale=en_US&access_token=" + token
  122.     res = requests.get(url)
  123.     realm_data = json.loads(res.content)
  124.     links = []
  125.     for i in (realm_data['connected_realms']):
  126.         links.append(i['href'].replace("?", "/auctions?") + "&locale=en_US&access_token=" + token)
  127.     return links
  128.  
  129.  
  130. # ends here
  131.  
  132. # Add your client ID and secret here, obtain both from https://develop.battle.net/access/.
  133. # Create a client, generate a secret, plug them into create_access_token('client id goes here', 'secret goes here').
  134.  
  135. response = create_access_token('client id here', 'secret here')
  136. token = response['access_token']
  137.  
  138. # ends here
  139.  
  140.  
  141. # Open up the realm_times dict from file:
  142. if os.path.exists("realm_timestamps.pickle"):
  143.     try:
  144.         with open("realm_timestamps.pickle", "rb") as f:
  145.             realm_dict = pickle.load(f)
  146.     except:
  147.         realm_dict = {}
  148. else:
  149.     realm_dict = {}
  150.     # ...or build it on initial scrape.
  151.  
  152.  
  153. # ends here
  154.  
  155. # This shows the api update time for each realm, you can comment this out if you want.
  156. # def sorter(e):
  157. #     t = realm_dict[e][0]
  158. #
  159. #     base_time = datetime(1980, 1, 1, 1, 1, 1)
  160. #     key_time = datetime.strptime(t, "%a, %d %b %Y %H:%M:%S %Z")
  161. #     difference = key_time - base_time
  162. #     seconds_in_day = 24 * 60 * 60
  163. #     x, y = divmod(difference.days * seconds_in_day + difference.seconds, 60)
  164. #     x = x + y
  165. #
  166. #     return x
  167. #
  168. #
  169. # keys = realm_dict.keys()
  170. # sort = sorted(realm_dict, key=sorter)
  171. # print(f"-- {len(sort)} Total Realms --")
  172. # for value in sort:
  173. #     print(f">> {value}, {realm_dict[value][0]}")
  174.  
  175. # ends here
  176.  
  177. print("Start Scan:")
  178.  
  179. # This cycles  the downloads of the realms as they go through their hourly updates:
  180. while 1:
  181.     items_to_print = []
  182.     realms = get_realm_urls(token)
  183.     realms_to_download = []
  184.     for realm in realms:
  185.         realmid = int(realm.split("?")[0].split("/")[-2])
  186.         if realmid not in realm_dict or (datetime.utcnow() - realm_dict[realmid][1]).seconds >= 3600:
  187.             realms_to_download.append(realm)
  188.     # print(realms_to_download)
  189.     # print(len(realms_to_download))
  190.     stime = time.time()
  191.     if len(realms_to_download) > 0:
  192.         download_realm_data(realms_to_download)
  193.     # print(datetime.now().strftime('%H:%M:%S'))
  194.  
Add Comment
Please, Sign In to add comment