Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import scrapy
- from scrapy.http.request import Request
- from scrapy.selector import Selector
- from scg_price.items import ScgPriceItem
- class StarcytigamesSpider(scrapy.Spider):
- name = 'starcitygames'
- allowed_domains = ['starcitygames.com']
- def __init__(self, *args, **kwargs):
- self.myurls = kwargs.get('myurls', [])
- super(StarcytigamesSpider, self).__init__(*args, **kwargs)
- def start_requests(self):
- for url in self.myurls:
- yield Request(url, self.parse)
- def parse(self, response):
- print('crawling')
- prices_item = ScgPriceItem()
- selector = Selector(text=response.text)
- prices = selector.xpath('//div[@class="hawk-results-item__options-table-cell hawk-results-item__options-table-cell--price childAttributes"]/text()').getall()
- valid_prices = [float(price[1::]) for price in prices]
- prices_item['max_price'] = max(valid_prices)
- prices_item['min_price'] = min(valid_prices)
- print(prices_item)
- return prices_item
- ===== bot.py ========
- @defer.inlineCallbacks
- def crawl(card_search: str):
- runner = CrawlerRunner(get_project_settings())
- yield runner.crawl('starcitygames', domain='starcitygames.com', myurls=[card_search])
- reactor.stop()
- @dp.message_handler()
- async def scryfall_find_card(message: types.Message):
- #<DOING SMTH>
- card_link = card_scg_link_form(response_json['name'])
- card_search = card_scg_search_form(response_json['name'])
- card_name = response_json['name']
- # run scrapy
- prices = []
- crawl(card_search)
- reactor.run()
- await message.reply_photo(response_json['image_uris']['normal'], caption=f'<a href="{card_link}">{card_name}</a>'
- , parse_mode='HTML')
Advertisement
Add Comment
Please, Sign In to add comment