Advertisement
4killU

Untitled

Jun 6th, 2022
1,143
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 4.81 KB | None | 0 0
  1. from database_files.X_Database import XboxDB
  2. from bs4 import BeautifulSoup
  3. from requests import cookies
  4. from datetime import datetime
  5. import fake_useragent
  6. import requests
  7. import json
  8. import os
  9. import re
  10.  
  11. xbox = XboxDB(r'D:\Python Projects\Xboxer-bot\database_files\xboxer_database.db')
  12.  
  13. user = fake_useragent.UserAgent().random
  14.  
  15. header = {
  16.     'user-agent': user
  17. }
  18.  
  19. cookies = {
  20.     'xnv2_currency_country': 'UA'
  21. }
  22.  
  23. general_newsblock_link = 'https://www.xbox-now.com/ru/news'
  24.  
  25.  
  26. def parse_game_dlc_links():
  27.     response = requests.get(general_newsblock_link, headers=header, cookies=cookies)
  28.     soup = BeautifulSoup(response.text, "lxml")
  29.  
  30.     search_news_blocks = soup.find('div', class_='news-entry-newstext')
  31.  
  32.     search_game_spoiler_panel = search_news_blocks.find_all('div', class_='panel-body')
  33.  
  34.     all_links_from_newsblock = []
  35.  
  36.     for tag_a in search_game_spoiler_panel:
  37.         a = tag_a.find_all('a')
  38.         for details in a:
  39.             all_links_from_newsblock.append(details['href'])
  40.  
  41.     # print(all_links_from_newsblock)
  42.     # print(len(all_links_from_newsblock))
  43.  
  44.     return all_links_from_newsblock
  45.  
  46.  
  47. def parsing_names_prices_from_links():
  48.     #parsed_url = parse_game_dlc_links()
  49.     parsed_url = ['https://www.xbox-now.com/ru/game/6188/watch-dogs2-deluxe-edition']
  50.  
  51.     total_parsed_result = []
  52.  
  53.     for url in parsed_url:
  54.  
  55.         game_details = {}
  56.         prices_dict = {}
  57.  
  58.         response = requests.post(url, headers=header, cookies=cookies)
  59.         soup = BeautifulSoup(response.text, "lxml")
  60.  
  61.         game_name_on_page = soup.find_all('h2')[0].text.strip()
  62.         game_details['game_name'] = game_name_on_page
  63.  
  64.         picture_url = soup.find(itemprop="image")['src']
  65.  
  66.         game_details['img'] = picture_url
  67.         game_details['url'] = url
  68.  
  69.         prices_rows = soup.select('.col-xs-4.col-sm-3')
  70.  
  71.         for rows in prices_rows:
  72.             try:
  73.                 if ' TRY' in rows.text.strip() or ' ARS' in rows.text.strip() or ' INR' in rows.text.strip():
  74.                     if 'On Sale' in rows.text.strip() or 'с GOLD ' in rows.text.strip():
  75.                         all_prices = rows.text.strip().replace(u'\xa0', u'').replace('\n', '').split(')')[1:]
  76.                         low_price = all_prices[0].strip().split(' UAH')
  77.                         prices_dict[float(low_price[0].strip())] = low_price[1]
  78.                     elif 'Обычная цена' in rows.text.strip() :
  79.                         all_prices = rows.text.strip().replace(u'\xa0', u'').replace('\n', '').split('Обычная цена')[1:]
  80.                         print(all_prices)
  81.                         print(all_prices[0])
  82.                         low_price = all_prices[0].strip().split(' RUB')
  83.                         print(low_price)
  84.                         print(low_price[0])
  85.                         prices_dict[low_price[0].strip()] = low_price[1]
  86.                     else:
  87.                         all_prices = rows.text.strip().replace(u'\xa0', u'').replace('\n', '').split(')')
  88.                         low_price = all_prices[0].strip().split(' UAH')
  89.                         prices_dict[float(low_price[0].strip())] = low_price[1]
  90.  
  91.                     min_price = min(prices_dict.keys())
  92.                     game_details['low_price'] = f'{min_price} UAH'
  93.                     if ' TRY' in prices_dict.get(min_price):
  94.                         game_details['region'] = 'Турция'
  95.                     elif ' ARS' in prices_dict.get(min_price):
  96.                         game_details['region'] = 'Аргентина'
  97.                     elif ' INR' in prices_dict.get(min_price):
  98.                         game_details['region'] = 'Индия'
  99.             except IndexError:
  100.                 pass
  101.             except ValueError:
  102.                 pass
  103.  
  104.         total_parsed_result.append(game_details)
  105.  
  106.     print(f'TOTAL RESULT : {len(total_parsed_result)}')
  107.  
  108.     day_today = datetime.now().strftime('%d-%m-%Y')
  109.  
  110.     with open(f'{day_today}.json', 'w', encoding="utf-8") as file:
  111.         json.dump(total_parsed_result, file, ensure_ascii=False)
  112.  
  113.     file_path = os.path.realpath(f'{day_today}.json')
  114.  
  115.     print(file_path)
  116.     return file_path
  117.  
  118. parsing_names_prices_from_links()
  119.  
  120. def search_new_deals():
  121.     general_news_link = 'https://www.xbox-now.com/ru/news'
  122.     response = requests.get(general_news_link, headers=header, cookies=cookies)
  123.     soup = BeautifulSoup(response.text, "lxml")
  124.  
  125.     news_entry_infoname = soup.find('div', class_='news-entry-info').find(itemprop="headline").text
  126.     last_deals_news_fromdb = xbox.select_deals_details()
  127.  
  128.     if not news_entry_infoname == last_deals_news_fromdb[-1][0]:
  129.         inert_last_deals = xbox.insert_deals_details(news_entry_infoname)
  130.         return True
  131.     else:
  132.         print('No new news, nothing to parse')
  133.         return False
  134.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement