Guest User

Untitled

a guest
Dec 21st, 2020
42
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.77 KB | None | 0 0
  1. import scrapy
  2. from scrapy.http.request import Request
  3. from scrapy.selector import Selector
  4. from scg_price.items import ScgPriceItem
  5.  
  6. class StarcytigamesSpider(scrapy.Spider):
  7.     name = 'starcitygames'
  8.     allowed_domains = ['starcitygames.com']
  9.  
  10.     def __init__(self, *args, **kwargs):
  11.         self.myurls = kwargs.get('myurls', [])
  12.         super(StarcytigamesSpider, self).__init__(*args, **kwargs)
  13.  
  14.     def start_requests(self):
  15.         for url in self.myurls:
  16.             yield Request(url, self.parse)
  17.  
  18.     def parse(self, response):
  19.         print('crawling')
  20.         prices_item = ScgPriceItem()
  21.         selector = Selector(text=response.text)
  22.         prices = selector.xpath('//div[@class="hawk-results-item__options-table-cell hawk-results-item__options-table-cell--price childAttributes"]/text()').getall()
  23.         valid_prices = [float(price[1::]) for price in prices]
  24.         prices_item['max_price'] = max(valid_prices)
  25.         prices_item['min_price'] = min(valid_prices)
  26.         print(prices_item)
  27.         return prices_item
  28.  
  29.  
  30. ===== bot.py ========
  31. @defer.inlineCallbacks
  32. def crawl(card_search: str):
  33.     runner = CrawlerRunner(get_project_settings())
  34.     yield runner.crawl('starcitygames', domain='starcitygames.com', myurls=[card_search])
  35.     reactor.stop()
  36.  
  37. @dp.message_handler()
  38. async def scryfall_find_card(message: types.Message):
  39.     #<DOING SMTH>
  40.     card_link = card_scg_link_form(response_json['name'])
  41.     card_search = card_scg_search_form(response_json['name'])
  42.  
  43.     card_name = response_json['name']
  44.     # run scrapy
  45.     prices = []
  46.     crawl(card_search)
  47.     reactor.run()
  48.     await message.reply_photo(response_json['image_uris']['normal'], caption=f'<a href="{card_link}">{card_name}</a>'
  49.                             , parse_mode='HTML')
Advertisement
Add Comment
Please, Sign In to add comment