Guest User

Untitled

a guest
Jan 19th, 2019
85
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.71 KB | None | 0 0
  1. # -*- coding: utf-8 -*-
  2. from rtcm_dj.web.models import RedSocial
  3. from rtcm_dj.web.robots.FacebookDataCollector import FanpageDataCollector
  4. from rtcm_dj.web.robots.TwitterDataCollector import TwAccountDataCollector
  5. from rtcm_dj.web.robots.YoutubeDataCollector import YtAccountDataCollector
  6. from rtcm_dj.web.bdutil.bddebug_utils import Logger
  7.  
  8. from datetime import datetime
  9. from time import sleep
  10. import threading
  11. import Queue
  12. from sys import exc_info
  13.  
  14. # CONFIG
  15. number_of_threads = 8
  16. log = "/home/logs/robots.log"
  17.  
  18. queue = Queue.Queue(maxsize=number_of_threads)
  19. logger = Logger(log)
  20.  
  21. class GatheringThread(threading.Thread):
  22.  
  23. def __init__(self):
  24. self._stop = False
  25. threading.Thread.__init__(self)
  26.  
  27. def run(self):
  28. logger.log("Starting thread " + self.name)
  29. while not self._stop :
  30. try :
  31. red = queue.get(True, 5000)
  32. try :
  33. name = red.__class__.__name__
  34. self.name = name + "#" + str(red.id)
  35. if name == 'TwitterAccountSN' :
  36. collector = TwAccountDataCollector(red.id, logger)
  37. elif name == "FacebookFanpageSN" :
  38. collector = FanpageDataCollector(red.id, logger)
  39. elif name == "YoutubeAccountSN" :
  40. collector = YtAccountDataCollector(red.id, logger)
  41. collector.iniciar_recoleccion()
  42. except :
  43. logger.log("Unhandled exception caught", "error", exc_info=exc_info())
  44. queue.task_done()
  45. except queue.Empty :
  46. self._stop = True
  47.  
  48.  
  49.  
  50. def run():
  51. try :
  52. logger.log("#############################")
  53. logger.log(" {0} ".format(datetime.now()))
  54. logger.log("#############################")
  55.  
  56. # start threads
  57. for x in xrange(number_of_threads) :
  58. t = GatheringThread().start()
  59.  
  60. # queue work
  61. redes = RedSocial.objects.filter()
  62. for red in redes :
  63. queue.put(red)
  64.  
  65. # wait for work to finish
  66. queue.join()
  67. x = 0
  68. for thread in threading.enumerate() :
  69. if x :
  70. thread._Thread__stop()
  71. x += 1
  72.  
  73. logger.log("Work done, threads stopping")
  74.  
  75. except KeyboardInterrupt :
  76. logger.log("KeyboardInterrupt. Killing all threads", "warning")
  77. x = 0
  78. for thread in threading.enumerate() :
  79. if x :
  80. thread._Thread__stop()
  81. x += 1
  82. logger.log("Threads killed", "warning")
Add Comment
Please, Sign In to add comment