spider settings

# -*- coding: utf-8 -*-

# Scrapy settings for ia_check project

BOT_NAME = 'ia_check'

SPIDER_MODULES = ['ia_check.spiders']
NEWSPIDER_MODULE = 'ia_check.spiders'
DOWNLOAD_HANDLERS = {'s3':None}

DOWNLOADER_MIDDLEWARES = {
'scrapy_splash.SplashCookiesMiddleware': 723,
'scrapy_crawlera.CrawleraMiddleware': 600,
'scrapyjs.SplashMiddleware': 725
}

SPLASH_URL = 'http://192.168.59.103:8050' # http://localhost:8050/

DUPEFILTER_CLASS = 'scrapyjs.SplashAwareDupeFilter'
HTTPCACHE_STORAGE = 'scrapyjs.SplashAwareFSCacheStorage'

CRAWLERA_ENABLED = True
CRAWLERA_APIKEY = '7ea1c5bc865849d4bbacba460e7122ac'

# DOWNLOAD_DELAY=3
# The download delay setting will honor only one of:
CONCURRENT_REQUESTS_PER_DOMAIN=16
CONCURRENT_REQUESTS_PER_IP=16