Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # -*- coding: utf-8 -*-
- from rtcm_dj.web.models import RedSocial
- from rtcm_dj.web.robots.FacebookDataCollector import FanpageDataCollector
- from rtcm_dj.web.robots.TwitterDataCollector import TwAccountDataCollector
- from rtcm_dj.web.robots.YoutubeDataCollector import YtAccountDataCollector
- from rtcm_dj.web.bdutil.bddebug_utils import Logger
- from datetime import datetime
- from time import sleep
- import threading
- import Queue
- from sys import exc_info
- # CONFIG
- number_of_threads = 8
- log = "/home/logs/robots.log"
- queue = Queue.Queue(maxsize=number_of_threads)
- logger = Logger(log)
- class GatheringThread(threading.Thread):
- def __init__(self):
- self._stop = False
- threading.Thread.__init__(self)
- def run(self):
- logger.log("Starting thread " + self.name)
- while not self._stop :
- try :
- red = queue.get(True, 5000)
- try :
- name = red.__class__.__name__
- self.name = name + "#" + str(red.id)
- if name == 'TwitterAccountSN' :
- collector = TwAccountDataCollector(red.id, logger)
- elif name == "FacebookFanpageSN" :
- collector = FanpageDataCollector(red.id, logger)
- elif name == "YoutubeAccountSN" :
- collector = YtAccountDataCollector(red.id, logger)
- collector.iniciar_recoleccion()
- except :
- logger.log("Unhandled exception caught", "error", exc_info=exc_info())
- queue.task_done()
- except queue.Empty :
- self._stop = True
- def run():
- try :
- logger.log("#############################")
- logger.log(" {0} ".format(datetime.now()))
- logger.log("#############################")
- # start threads
- for x in xrange(number_of_threads) :
- t = GatheringThread().start()
- # queue work
- redes = RedSocial.objects.filter()
- for red in redes :
- queue.put(red)
- # wait for work to finish
- queue.join()
- x = 0
- for thread in threading.enumerate() :
- if x :
- thread._Thread__stop()
- x += 1
- logger.log("Work done, threads stopping")
- except KeyboardInterrupt :
- logger.log("KeyboardInterrupt. Killing all threads", "warning")
- x = 0
- for thread in threading.enumerate() :
- if x :
- thread._Thread__stop()
- x += 1
- logger.log("Threads killed", "warning")
Add Comment
Please, Sign In to add comment