Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import requests
- import concurrent.futures
- from Items import itemName
- from Realms import realmNames
- from Pets import petName
- import time
- import os
- from datetime import datetime
- import pickle
- import json
- # how to install:
- # If you don't have a file for any of the imports above this, you need to install it through cmd.
- # If you try to run the program and something is missing or it doesn't run, install what's missing and try again.
- # Press your windows key + r, type "cmd" without quotes, type in the following:
- # pip install python
- # pip install pickle
- # pip install time
- # To run the program, install any IDLE that supports Python, I recommend Pycharm: https://www.jetbrains.com/pycharm/
- # Make sure all the associated files are located in the same file as your main file or else it won't work.
- # change this if your PC can't handle 83 threads:
- MAX_THREADS = 83
- itemDict = {}
- petDict = {}
- # This is used for your dictionaries:
- for block in itemName:
- for item in itemName[block]:
- itemDict[item] = {'price': block, 'name': itemName[block][item]}
- for block in petName:
- for item in petName[block]:
- petDict[item] = {'price': block, 'name': petName[block][item]}
- # This is the multithreading section, it controls the download of urls and the search function:
- def download_url(url):
- global realm_dict
- if int(url.split("?")[0].split("/")[-2]) in realmNames:
- realm_name = realmNames[int(url.split("?")[0].split("/")[-2])]
- else:
- realm_name = "unknown"
- try:
- realmid = int(url.split("?")[0].split("/")[-2])
- if realmid in realm_dict:
- resp = requests.get(url, headers={'If-Modified-Since': realm_dict[realmid][0]})
- else:
- resp = requests.get(url)
- if resp.status_code == 304:
- pass # print("not modified")
- else:
- data = {}
- res = requests.get(url)
- auctions = res.json()['auctions']
- realm_dict[realmid] = [res.headers['Last-Modified'],
- datetime.strptime(res.headers['Last-Modified'][4:-4].strip(), '%d %b %Y %H:%M:%S')]
- with open("realm_timestamps.pickle", "wb") as f:
- pickle.dump(realm_dict, f)
- for i in auctions:
- if 'unit_price' in i:
- price = i['unit_price']
- elif 'buyout' in i:
- price = i['buyout']
- else:
- price = 0 # should never happen
- item_id = i["item"]['id']
- if price > 0:
- if item_id in itemDict:
- if price <= (itemDict[item_id]['price'] * 10000):
- try:
- print(f"- ITEM - {realm_name}: {itemDict[item_id]['name']} | {price / 10000}g")
- except Exception as msg:
- print(str(msg))
- elif 'pet_species_id' in i['item']:
- item_id = i['item']['pet_species_id']
- if item_id in petDict:
- if price <= (petDict[item_id]['price'] * 10000):
- try:
- if int(url.split("?")[0].split("/")[-2]) in realmNames:
- print(f"- ITEM - {realm_name}: {petDict[item_id]['name']} | {price / 10000}g")
- else:
- print(f"- ITEM - Unknown Realm {petDict[item_id]['name']} | {price / 10000}g")
- except Exception as msg:
- print("pet id missing -----(" + str(msg) + ")" + str(item_id) + "<<")
- except Exception as msg:
- print(str(msg))
- return data
- # Thread controller:
- def download_realm_data(urls):
- threads = min(MAX_THREADS, len(urls))
- with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as executor:
- executor.map(download_url, urls)
- # End multithreading functions
- # This creates your access token:
- def create_access_token(client_id, client_secret, region='us'):
- data = {'grant_type': 'client_credentials'}
- response = requests.post('https://%s.battle.net/oauth/token' % region, data=data, auth=(client_id, client_secret))
- return response.json()
- # ends here
- # This grabs urls for downloading:
- def get_realm_urls(token):
- url = "https://us.api.blizzard.com/data/wow/connected-realm/index?namespace=dynamic-us&locale=en_US&access_token=" + token
- res = requests.get(url)
- realm_data = json.loads(res.content)
- links = []
- for i in (realm_data['connected_realms']):
- links.append(i['href'].replace("?", "/auctions?") + "&locale=en_US&access_token=" + token)
- return links
- # ends here
- # Add your client ID and secret here, obtain both from https://develop.battle.net/access/.
- # Create a client, generate a secret, plug them into create_access_token('client id goes here', 'secret goes here').
- response = create_access_token('client id here', 'secret here')
- token = response['access_token']
- # ends here
- # Open up the realm_times dict from file:
- if os.path.exists("realm_timestamps.pickle"):
- try:
- with open("realm_timestamps.pickle", "rb") as f:
- realm_dict = pickle.load(f)
- except:
- realm_dict = {}
- else:
- realm_dict = {}
- # ...or build it on initial scrape.
- # ends here
- # This shows the api update time for each realm, you can comment this out if you want.
- # def sorter(e):
- # t = realm_dict[e][0]
- #
- # base_time = datetime(1980, 1, 1, 1, 1, 1)
- # key_time = datetime.strptime(t, "%a, %d %b %Y %H:%M:%S %Z")
- # difference = key_time - base_time
- # seconds_in_day = 24 * 60 * 60
- # x, y = divmod(difference.days * seconds_in_day + difference.seconds, 60)
- # x = x + y
- #
- # return x
- #
- #
- # keys = realm_dict.keys()
- # sort = sorted(realm_dict, key=sorter)
- # print(f"-- {len(sort)} Total Realms --")
- # for value in sort:
- # print(f">> {value}, {realm_dict[value][0]}")
- # ends here
- print("Start Scan:")
- # This cycles the downloads of the realms as they go through their hourly updates:
- while 1:
- items_to_print = []
- realms = get_realm_urls(token)
- realms_to_download = []
- for realm in realms:
- realmid = int(realm.split("?")[0].split("/")[-2])
- if realmid not in realm_dict or (datetime.utcnow() - realm_dict[realmid][1]).seconds >= 3600:
- realms_to_download.append(realm)
- # print(realms_to_download)
- # print(len(realms_to_download))
- stime = time.time()
- if len(realms_to_download) > 0:
- download_realm_data(realms_to_download)
- # print(datetime.now().strftime('%H:%M:%S'))
Add Comment
Please, Sign In to add comment