SHOW:
|
|
- or go back to the newest paste.
| 1 | import os | |
| 2 | - | from bs4 import BeautifulSoup |
| 2 | + | from functions.supportfunctions import dump |
| 3 | import requests | |
| 4 | - | headers = {'user-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0',
|
| 4 | + | from bs4 import BeautifulSoup as bs |
| 5 | - | 'BX-ACTION-TYPE':'get_dynamic', |
| 5 | + | import re |
| 6 | - | 'BX-CACHE-MODE':'HTMLCACHE', |
| 6 | + | |
| 7 | - | } |
| 7 | + | # url1 = 'https://www.vfbank.ru/fizicheskim-licam/monety/pamyatnye-monety/' |
| 8 | # url2 = 'https://www.vfbank.ru/fizicheskim-licam/monety/' | |
| 9 | - | url = 'https://www.vfbank.ru/fizicheskim-licam/monety/pamyatnye-monety/' |
| 9 | + | |
| 10 | # urls = [url1, url2] | |
| 11 | - | def parsing(html): |
| 11 | + | |
| 12 | - | start = html.find('bxdynamic_moneti_inner')
|
| 12 | + | def parsing_pam(url): |
| 13 | - | end = html.find("'HASH':'844584f9f4f7',")
|
| 13 | + | cat_number = [] |
| 14 | buy_price = [] | |
| 15 | - | data = html[start:end].replace('\\n','').replace('\\','')
|
| 15 | + | sell_price = [] |
| 16 | ||
| 17 | - | soup = BeautifulSoup(data,"html.parser") |
| 17 | + | start = url.find('bxdynamic_moneti_inner')
|
| 18 | - | coins = soup.find_all('div',class_='col col--lg-6')
|
| 18 | + | end = url.find("'HASH':'844584f9f4f7',")
|
| 19 | - | for coin in coins: |
| 19 | + | |
| 20 | - | title = coin.find('a').text.strip()
|
| 20 | + | data = url[start:end].replace('\\n', '').replace('\\', '')
|
| 21 | - | _id = coin.find('a').get('data-id')
|
| 21 | + | soup = bs(data, "lxml") |
| 22 | - | print(title,_id) |
| 22 | + | coins = soup.find_all('div', class_='col col--lg-6')
|
| 23 | for coin in coins: | |
| 24 | - | response = requests.get(url,headers=headers) |
| 24 | + | |
| 25 | - | parsing(response.text) |
| 25 | + | cat_num = coin.find_all('span', class_="coin-line-detail__title")
|
| 26 | price = coin.find_all('span', class_="coin-line-detail__desc")
| |
| 27 | ||
| 28 | cat_number.append(cat_num[1].text.replace('№', '').strip())
| |
| 29 | buy_sell = price[1].text.strip().replace(' ','').replace('₽','').split('/')
| |
| 30 | buy_price.append(float(buy_sell[0])) | |
| 31 | sell_price.append(float(buy_sell[1])) | |
| 32 | ||
| 33 | cont = {"sell": [{"cat_number": c, "price": s}
| |
| 34 | for c, s in zip(cat_number, sell_price)], | |
| 35 | "buy": [{"cat_number": c, "price": b}
| |
| 36 | for c, b in zip(cat_number, buy_price)]} | |
| 37 | print(cont) | |
| 38 | return cont | |
| 39 | ||
| 40 | def parsing_inv(url): | |
| 41 | ||
| 42 | sell = [] | |
| 43 | buy = [] | |
| 44 | ||
| 45 | re_sell_buy = re.compile(r'Продажа:.*?(\d+\s?\d{,3}).*?'
| |
| 46 | r'Покупка:.*?(\d+\s?\d{,3}).*?', re.DOTALL)
| |
| 47 | re_catnum = re.compile(r'Каталожный номер:.+?(\d{4}-\d{4})', re.DOTALL)
| |
| 48 | ||
| 49 | soup = bs(url, "lxml") | |
| 50 | coins = soup.find_all('h2', class_='custom-widget-title coin__title')
| |
| 51 | ||
| 52 | coin_name = [i.text.strip() for i in coins] | |
| 53 | ||
| 54 | for i in re_sell_buy.findall(url): | |
| 55 | # print(i) | |
| 56 | sell.append(float(i[0].strip().replace(' ', '').replace('\n', '').replace('\t', '')))
| |
| 57 | ||
| 58 | buy.append(float(i[1].strip().replace(' ', '').replace('\n', '').replace('\t', '')))
| |
| 59 | ||
| 60 | cat_number = [i for i in re_catnum.findall(url)] | |
| 61 | ||
| 62 | cont = {"sell": [{'coin_name': cn, "cat_number": c, "price": s}
| |
| 63 | for cn, c, s in zip(coin_name, cat_number, sell)], | |
| 64 | "buy": [{'coin_name': cn, "cat_number": c, "price": s}
| |
| 65 | for cn, c, s in zip(coin_name, cat_number, sell)] | |
| 66 | } | |
| 67 | ||
| 68 | # print(cont) | |
| 69 | return cont | |
| 70 | ||
| 71 | def parser(urls): | |
| 72 | cont = dict() | |
| 73 | for url in urls: | |
| 74 | if url == 'https://www.vfbank.ru/fizicheskim-licam/monety/pamyatnye-monety/': | |
| 75 | headers = {'user-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0',
| |
| 76 | 'BX-ACTION-TYPE': 'get_dynamic', | |
| 77 | 'BX-CACHE-MODE': 'HTMLCACHE', | |
| 78 | } | |
| 79 | response = requests.get(url, headers=headers) | |
| 80 | cont.update(parsing_pam(response.text)) | |
| 81 | elif url == 'https://www.vfbank.ru/fizicheskim-licam/monety/': | |
| 82 | response = requests.get(url) | |
| 83 | parsed = parsing_inv(response.text) | |
| 84 | for k in parsed.keys(): | |
| 85 | if k in cont: | |
| 86 | cont[k].extend(parsed[k]) | |
| 87 | else: | |
| 88 | cont[k] = parsed[k] | |
| 89 | return cont | |
| 90 | ||
| 91 | ||
| 92 | url1 = 'https://www.vfbank.ru/fizicheskim-licam/monety/pamyatnye-monety/' | |
| 93 | url2 = 'https://www.vfbank.ru/fizicheskim-licam/monety/' | |
| 94 | ||
| 95 | urls = [url1, url2] | |
| 96 | ||
| 97 | cont = parser(urls) | |
| 98 | ||
| 99 | print(cont) |