Advertisement
Guest User

Untitled

a guest
May 27th, 2019
121
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 0.59 KB | None | 0 0
  1.  def start_requests(self):
  2.         for url in self.start_urls:
  3.             print(url)
  4.             yield Request(url=url, method='GET', meta=self.meta, encoding='utf-8', dont_filter=True, callback=self.parse, errback=self.error_back)
  5.         self.spider_idle(FrenzyspiderSpider)
  6.  
  7.     def spider_idle(self, spider):
  8.         print('sleep')
  9.         sleep(1)
  10.         for r in self.start_requests():
  11.             self.crawler.engine.schedule(r, FrenzyspiderSpider)
  12.         raise DontCloseSpider
  13.  
  14.     def parse(self, response):
  15.         data = json.loads(response.text)
  16.         print(self.parsed)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement