Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- class CustomRetryMiddleware(RetryMiddleware):
- def process_response(self, request, response, spider):
- if request.meta.get('dont_retry', False):
- return response
- if 'showcaptcha?' in response.url:
- reason = 'capcha'
- original_url = request.meta['original_url']
- new_request = request.replace(url=original_url)
- return self._retry(new_request, reason, spider) or response
- return response
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement