class SFAProxyMiddleware(object):
    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler.settings)

    def __init__(self, settings):

        self.packetstream_proxies = [
            settings.get("PS_PROXY_USA"),
            settings.get("PS_PROXY_CA"),
            settings.get("PS_PROXY_IT"),
            settings.get("PS_PROXY_GLOBAL"),
        ]

        self.unlimited_proxies = [
            settings.get("UNLIMITED_PROXY_1"),
            settings.get("UNLIMITED_PROXY_2"),
            # settings.get("UNLIMITED_PROXY_3"),
            # settings.get("UNLIMITED_PROXY_4"),
            # settings.get("UNLIMITED_PROXY_5"),
            # settings.get("UNLIMITED_PROXY_6"),
        ]

    def add_proxy(self, request, host):
        request.meta["proxy"] = host

    def process_request(self, request, spider):
        retries = request.meta.get("retry_times", 0)

        if retries <= 8:
            self.add_proxy(request, random.choice(self.unlimited_proxies))
            spider.logger.info("In unlimiited proxies block")
            return None
        spider.logger.info("Using packetstream proxies now")

        self.add_proxy(request, random.choice(self.packetstream_proxies))
        return None

    def process_response(self, request, response, spider):

        return response