Advertisement
Guest User

Untitled

a guest
Feb 19th, 2020
100
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 2.09 KB | None | 0 0
  1. import time
  2. import requests
  3. from bs4 import BeautifulSoup
  4. import lxml
  5. from MatchScrape import MatchScrape
  6.  
  7.  
  8. # TODO add several queries based on different regions than just europe west.
  9.  
  10. class Main:
  11.     def __init__(self):
  12.         self.soup = BeautifulSoup(self.getpage(), 'lxml')
  13.         self.matches = []
  14.  
  15.     def main(self):
  16.         self.matchscrape()
  17.  
  18.     def getpage(self):
  19.         result = requests.get(
  20.             "https://www.dotabuff.com/matches?game_mode=all_pick&lobby_type=ranked_matchmaking&region=europe_west&skill_bracket=very_high_skill",
  21.             headers={'User-agent': 'your bot 0.1'})
  22.         if result.status_code == 200:
  23.             print("found page")
  24.             return result.content
  25.         else:
  26.             print("did not find page, waiting 1 second")
  27.             time.sleep(1000)
  28.             self.getpage()
  29.  
  30.     def refreshpage(self):
  31.         self.soup = BeautifulSoup(self.getpage(), 'lxml')
  32.  
  33.     def matchscrape(self):
  34.         self.matches = []
  35.         # get all match IDs and "minutes ago" with beatifulsoup.
  36.         # get the table entries
  37.         matchTable = self.soup.find("table").find("tbody").find_all("tr")
  38.         # get the id and time associated
  39.         for tr in matchTable:
  40.             cols = tr.find_all('td')
  41.             # due to website layout, we only need the first col.
  42.             matchColumn = cols[0]
  43.             matchId = matchColumn.find('a').text
  44.             # print(matchId)
  45.             matchDate = matchColumn.find('time')['datetime']
  46.             # print(matchDate)
  47.             match = MatchScrape(matchId, matchDate)
  48.             # just incase, lets try to not add several of same matchId.
  49.             if not any(matc.matchId == matchId for matc in self.matches):
  50.                 self.matches.append(match)
  51.  
  52.     def getmatchtable(self):
  53.         return self.matches
  54.  
  55.  
  56. if __name__ == '__main__':
  57.     m = Main()
  58.     m.main()
  59.  
  60. class MatchScrape:
  61.     date = ""
  62.     matchId = ""
  63.     region = ""
  64.     downloaded = False
  65.  
  66.     def __init__(self, matchid, date):
  67.         self.matchId = matchid
  68.         self.date = date
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement