Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def main(target_year):
- project = os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0]
- iso_run_date = datetime.date.today().isoformat()
- logger = utils.get_logger(project, iso_run_date)
- scraping.run(project, iso_run_date, target_year)
- class UTCFormatter(logging.Formatter):
- converter = time.gmtime
- def get_logger(project, iso_run_date):
- ip_address_param = 'ip'
- logger = logging.getLogger(project)
- logger.setLevel(logging.DEBUG)
- file_handler = logging.FileHandler(os.path.abspath(os.path.join(
- 'log', '{}_{}.log'.format(project, iso_run_date))))
- file_handler.setLevel(logging.DEBUG)
- formatter = UTCFormatter(
- fmt=('[%(asctime)s.%(msecs)03dZ] %({})s %(name)s %(levelname)s: '
- '%(message)s').format(ip_address_param),
- datefmt='%Y-%m-%dT%H:%M:%S')
- file_handler.setFormatter(formatter)
- logger.addHandler(file_handler)
- logger = logging.LoggerAdapter(
- logger, {ip_address_param: socket.gethostbyname(socket.gethostname())})
- return logger
- @twisted.internet.defer.inlineCallbacks
- def crawl(crawler_process, project, iso_run_date, target_year):
- yield crawler_process.crawl(project, iso_run_date, target_year)
- def run(project, iso_run_date, target_year):
- os.environ.setdefault(
- 'SCRAPY_SETTINGS_MODULE', 'scraping.scraping.settings')
- crawler_process = scrapy.crawler.CrawlerProcess(
- scrapy.utils.project.get_project_settings())
- crawl(crawler_process, project, iso_run_date, target_year)
- crawler_process.start()
- self.logger.debug('Test')
- --- Logging error ---
- Traceback (most recent call last):
- File "/usr/lib/python3.5/logging/__init__.py", line 980, in emit
- msg = self.format(record)
- File "/usr/lib/python3.5/logging/__init__.py", line 830, in format
- return fmt.format(record)
- File "/usr/lib/python3.5/logging/__init__.py", line 570, in format
- s = self.formatMessage(record)
- File "/usr/lib/python3.5/logging/__init__.py", line 539, in formatMessage
- return self._style.format(record)
- File "/usr/lib/python3.5/logging/__init__.py", line 383, in format
- return self._fmt % record.__dict__
- KeyError: 'ip'
- Call stack:
- File "XXXXX.py", line 105, in <module>
- main(target_year)
- File "XXXXX.py", line 23, in main
- scraping.run(project, iso_run_date, target_year)
- File "/home/XYZ/virtualenvs/scraping/project/scraping/__init__.py", line 27, in run
- crawler_process.start()
- File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/scrapy/crawler.py", line 291, in start
- reactor.run(installSignalHandlers=False) # blocking call
- File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/twisted/internet/base.py", line 1261, in run
- self.mainLoop()
- File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/twisted/internet/base.py", line 1270, in mainLoop
- self.runUntilCurrent()
- File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/twisted/internet/base.py", line 896, in runUntilCurrent
- call.func(*call.args, **call.kw)
- File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/scrapy/utils/reactor.py", line 41, in __call__
- return self._func(*self._a, **self._kw)
- File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/scrapy/core/engine.py", line 127, in _next_request
- request = next(slot.start_requests)
- File "/home/XYZ/virtualenvs/scraping/project/scraping/scraping/spiders/XXXXX.py", line 47, in start_requests
- self.logger.debug('Test')
- Message: 'Test'
- Arguments: ()
- def main(target_year):
- project = os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0]
- iso_run_date = datetime.date.today().isoformat()
- logger = utils.get_logger(project, iso_run_date)
- def get_logger(project, iso_run_date):
- logging.basicConfig(
- filename=os.path.abspath(os.path.join('log', '{0}_{1}.log'.format(project, iso_run_date))),
- format='[%(asctime)s.%(msecs)03dZ] {0} %(name)s %(levelname)s: %(message)s'.format(socket.gethostbyname(socket.gethostname())),
- datefmt='%Y-%m-%dT%H:%M:%S',
- level=logging.DEBUG)
- logging.Formatter.converter = time.gmtime
- return logging.getLogger(project)
- @twisted.internet.defer.inlineCallbacks
- def crawl(crawler_process, project, iso_run_date, target_year):
- yield crawler_process.crawl(project, iso_run_date, target_year)
- def run(project, iso_run_date, target_year):
- os.environ.setdefault(
- 'SCRAPY_SETTINGS_MODULE', 'scraping.scraping.settings')
- crawler_process = scrapy.crawler.CrawlerProcess(
- scrapy.utils.project.get_project_settings())
- crawl(crawler_process, project, iso_run_date, target_year)
- crawler_process.start()
Add Comment
Please, Sign In to add comment