Guest User

Untitled

a guest
Jun 18th, 2018
64
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.63 KB | None | 0 0
  1. def main(target_year):
  2. project = os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0]
  3. iso_run_date = datetime.date.today().isoformat()
  4. logger = utils.get_logger(project, iso_run_date)
  5.  
  6. scraping.run(project, iso_run_date, target_year)
  7.  
  8. class UTCFormatter(logging.Formatter):
  9. converter = time.gmtime
  10.  
  11.  
  12. def get_logger(project, iso_run_date):
  13. ip_address_param = 'ip'
  14. logger = logging.getLogger(project)
  15. logger.setLevel(logging.DEBUG)
  16. file_handler = logging.FileHandler(os.path.abspath(os.path.join(
  17. 'log', '{}_{}.log'.format(project, iso_run_date))))
  18. file_handler.setLevel(logging.DEBUG)
  19. formatter = UTCFormatter(
  20. fmt=('[%(asctime)s.%(msecs)03dZ] %({})s %(name)s %(levelname)s: '
  21. '%(message)s').format(ip_address_param),
  22. datefmt='%Y-%m-%dT%H:%M:%S')
  23. file_handler.setFormatter(formatter)
  24. logger.addHandler(file_handler)
  25. logger = logging.LoggerAdapter(
  26. logger, {ip_address_param: socket.gethostbyname(socket.gethostname())})
  27. return logger
  28.  
  29. @twisted.internet.defer.inlineCallbacks
  30. def crawl(crawler_process, project, iso_run_date, target_year):
  31. yield crawler_process.crawl(project, iso_run_date, target_year)
  32.  
  33.  
  34. def run(project, iso_run_date, target_year):
  35. os.environ.setdefault(
  36. 'SCRAPY_SETTINGS_MODULE', 'scraping.scraping.settings')
  37. crawler_process = scrapy.crawler.CrawlerProcess(
  38. scrapy.utils.project.get_project_settings())
  39. crawl(crawler_process, project, iso_run_date, target_year)
  40. crawler_process.start()
  41.  
  42. self.logger.debug('Test')
  43.  
  44. --- Logging error ---
  45. Traceback (most recent call last):
  46. File "/usr/lib/python3.5/logging/__init__.py", line 980, in emit
  47. msg = self.format(record)
  48. File "/usr/lib/python3.5/logging/__init__.py", line 830, in format
  49. return fmt.format(record)
  50. File "/usr/lib/python3.5/logging/__init__.py", line 570, in format
  51. s = self.formatMessage(record)
  52. File "/usr/lib/python3.5/logging/__init__.py", line 539, in formatMessage
  53. return self._style.format(record)
  54. File "/usr/lib/python3.5/logging/__init__.py", line 383, in format
  55. return self._fmt % record.__dict__
  56. KeyError: 'ip'
  57. Call stack:
  58. File "XXXXX.py", line 105, in <module>
  59. main(target_year)
  60. File "XXXXX.py", line 23, in main
  61. scraping.run(project, iso_run_date, target_year)
  62. File "/home/XYZ/virtualenvs/scraping/project/scraping/__init__.py", line 27, in run
  63. crawler_process.start()
  64. File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/scrapy/crawler.py", line 291, in start
  65. reactor.run(installSignalHandlers=False) # blocking call
  66. File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/twisted/internet/base.py", line 1261, in run
  67. self.mainLoop()
  68. File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/twisted/internet/base.py", line 1270, in mainLoop
  69. self.runUntilCurrent()
  70. File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/twisted/internet/base.py", line 896, in runUntilCurrent
  71. call.func(*call.args, **call.kw)
  72. File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/scrapy/utils/reactor.py", line 41, in __call__
  73. return self._func(*self._a, **self._kw)
  74. File "/home/XYZ/virtualenvs/scraping/lib/python3.5/site-packages/scrapy/core/engine.py", line 127, in _next_request
  75. request = next(slot.start_requests)
  76. File "/home/XYZ/virtualenvs/scraping/project/scraping/scraping/spiders/XXXXX.py", line 47, in start_requests
  77. self.logger.debug('Test')
  78. Message: 'Test'
  79. Arguments: ()
  80.  
  81. def main(target_year):
  82. project = os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0]
  83. iso_run_date = datetime.date.today().isoformat()
  84. logger = utils.get_logger(project, iso_run_date)
  85.  
  86. def get_logger(project, iso_run_date):
  87. logging.basicConfig(
  88. filename=os.path.abspath(os.path.join('log', '{0}_{1}.log'.format(project, iso_run_date))),
  89. format='[%(asctime)s.%(msecs)03dZ] {0} %(name)s %(levelname)s: %(message)s'.format(socket.gethostbyname(socket.gethostname())),
  90. datefmt='%Y-%m-%dT%H:%M:%S',
  91. level=logging.DEBUG)
  92. logging.Formatter.converter = time.gmtime
  93. return logging.getLogger(project)
  94.  
  95. @twisted.internet.defer.inlineCallbacks
  96. def crawl(crawler_process, project, iso_run_date, target_year):
  97. yield crawler_process.crawl(project, iso_run_date, target_year)
  98.  
  99.  
  100. def run(project, iso_run_date, target_year):
  101. os.environ.setdefault(
  102. 'SCRAPY_SETTINGS_MODULE', 'scraping.scraping.settings')
  103. crawler_process = scrapy.crawler.CrawlerProcess(
  104. scrapy.utils.project.get_project_settings())
  105. crawl(crawler_process, project, iso_run_date, target_year)
  106. crawler_process.start()
Add Comment
Please, Sign In to add comment