Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def start_requests(self):
- for url in self.start_urls:
- print(url)
- yield Request(url=url, method='GET', meta=self.meta, encoding='utf-8', dont_filter=True, callback=self.parse, errback=self.error_back)
- self.spider_idle(FrenzyspiderSpider)
- def spider_idle(self, spider):
- print('sleep')
- sleep(1)
- for r in self.start_requests():
- self.crawler.engine.schedule(r, FrenzyspiderSpider)
- raise DontCloseSpider
- def parse(self, response):
- data = json.loads(response.text)
- print(self.parsed)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement