Guest User

Untitled

a guest
Dec 11th, 2010
205
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 8.15 KB | None | 0 0
  1. import gevent
  2. import logging
  3. import time
  4. import uuid
  5.  
  6. from random import randint
  7. from gevent import queue, pool, monkey, sleep
  8.  
  9. class with_timeout(object):
  10.     def __init__(self, timeout, callback=None):
  11.         '''Load timeout settings to decorator
  12.        timeout -- seconds value
  13.        callback -- if timeout reached and callback is set, will be fired with function arguments
  14.        exception -- action to be taken if decorated function running to long, options are following:
  15.            + False -> will supress any error
  16.            + None -> will raise gevent.Timeout exception
  17.            + CustomError -> will raise CustomError exception
  18.        Usage:
  19.        @Timeout(5) # 5 seconds to run this task
  20.        def foo(*args, **kwargs):
  21.            ...
  22.  
  23.        '''
  24.         self.timeout = timeout
  25.         self.callback = callback
  26.  
  27.     def __call__(self, f):
  28.         '''Decorate requested function'''
  29.         def wrapped(*args):
  30.             try:
  31.                 with gevent.Timeout(self.timeout, None):
  32.                     return f(*args)
  33.             except gevent.Timeout:
  34.                 if self.callback is not None:
  35.                     return self.callback(*args, **{'func': f.__name__})
  36.         return wrapped
  37.  
  38. class DispatchablePoolError(Exception): pass
  39. class HandlerError(DispatchablePoolError): pass
  40. class HandlerMissingError(HandlerError): pass
  41. class HandlerTimeoutError(HandlerError): pass
  42.  
  43. class DispatchablePool(pool.Pool):
  44.     '''This is gevent.Pool implementation with concurence task dispatch functionality.
  45.    You can subclass this class, build your own logic and then simply
  46.    call to instance.dispatch(tasks) in order to process your tasks asynchronously
  47.  
  48.    '''
  49.     def __init__(self, size=1, handlers=None, logger=None, apply_patch=True):
  50.         '''Pool initialization:
  51.        size -- how much asynchronicity you need?
  52.        handlers -- dictionary of custom task handlers, these handlers
  53.        has higher priority than thoose defined in class object itself.
  54.        Key of dictionary element maps to task name.
  55.        logger -- logging object, None value disables this functionality.
  56.  
  57.        '''
  58.         self.handlers = handlers
  59.         if handlers is None:
  60.             self.handlers = {}
  61.  
  62.         self.size = size
  63.         self.queue = queue.PriorityQueue()
  64.         self.logger = logger
  65.         if apply_patch:
  66.             monkey.patch_all()
  67.         self.start_time = 0
  68.         self.end_time = 0
  69.         self.tasks_done = False
  70.  
  71.         pool.Pool.__init__(self, size=self.size)
  72.  
  73.     def debug(self, message):
  74.         '''Log message via logger object'''
  75.         if self.logger:
  76.             self.logger.debug(message)
  77.  
  78.     def get_stats(self):
  79.         '''Current status of jobs'''
  80.         if self.tasks_done:
  81.             return "Jobs done in {:.3}s".format(self.end_time - self.start_time)
  82.         return "Jobs are not finished yet"
  83.  
  84.     def dispatch(self, tasks):
  85.         '''Run defined asynchronous tasks in an endless loop
  86.        tasks -- iterator object of tasks.
  87.        Single task typically looks like (priority, task, ...)
  88.  
  89.        '''
  90.         self.start_time = time.time()
  91.         for _task in tasks:
  92.             self.queue.put(_task)
  93.         while True:
  94.             while self.free_count():
  95.                 try:
  96.                     task = self.queue.get_nowait()
  97.                 except queue.Empty:
  98.                     break
  99. #                self.debug("Got task from queue: {}".format(task[:3]))
  100.                 handler = getattr(self, task[1], None)
  101.                 if task[1] in self.handlers:
  102.                     handler = self.handlers[task[1]]
  103.                 if handler is None:
  104.                     raise HandlerMissingError, \
  105.                     "Required handler for task {} not defined".format(task[1])
  106.                 self.spawn(handler, *(task[2:]))
  107. #                self.debug("Started task: {}".format(task[1]))
  108.             if self.free_count() == self.size:
  109.                 self.end_time = time.time()
  110.                 self.tasks_done = True
  111.                 break
  112.             sleep(1)
  113.  
  114. def image_handler_failed(*task, **kwargs):
  115.     '''Update images count awaiting processing at current document
  116.    Due to timeout we should decrease its number by one
  117.    '''
  118.     document = task[2]
  119.     document['images_cnt'] -= 1
  120.     processing.flush_results()
  121.     register_timeout_event(*task, **kwargs)
  122.  
  123. def register_timeout_event(*task, **kwargs):
  124.     '''General callback for timed out handlers'''
  125.     pool = task[0]
  126.     pool.debug("{} task timed out at {}".format(kwargs.get('func'), task[1]))
  127.  
  128. class RssProcessingPool(DispatchablePool):
  129.     # TODO: Make use of gevent.Timeout in content handlers, I'm afraid I can't use Timeout decorator :(
  130.     '''Use DisplatchablePool to process multiple RSS feeds in async mode.
  131.    Results are flushed to storage as soon as ready.
  132.  
  133.    '''
  134.     def __init__(self, **kwargs):
  135.         '''Setup pool
  136.        **kwargs -- same params as at superclass
  137.  
  138.        '''
  139.         self.results = []
  140.         DispatchablePool.__init__(self, **kwargs)
  141.  
  142.     def flush_results(self):
  143.         '''Save ready documents to db, free up memory
  144.        Ready document does not have any images scheduled to process.
  145.  
  146.        '''
  147.         for i, doc in enumerate(self.results):
  148.             if not doc['images_cnt']:
  149.                 self.debug("Document {} saved on disk".format(doc['content']))
  150.                 del self.results[i]
  151.  
  152.  
  153.     @with_timeout(1, image_handler_failed)
  154.     def image_handler(self, *task):
  155.         '''Complete docs at real implementation!
  156.        Process image. This handler *knows* about document its working with, thats very usefull for consistency
  157.        of document object.
  158.  
  159.        '''
  160.         self.debug('image_handler on {} started'.format(task[0]))
  161.         document = task[1]
  162.         document['images'].append(task[0])
  163.         sleep(randint(0, 5))
  164.         # decrease number of possible images for this documents that's are awaiting to check
  165.         document['images_cnt'] -= 1
  166.         self.flush_results()
  167.         self.debug('image_handler on {} completed'.format(task[0]))
  168.  
  169.     @with_timeout(2, register_timeout_event)
  170.     def document_handler(self, *task):
  171.         '''Complete docs at real implementation
  172.        Document Handler used to parse single documents from rss feeds, this method
  173.        analyzes document content and *could* run additional image rertival task.
  174.        Something interesting happens here - we setup document and then we pass it to image hadler,
  175.        so image handler will *know* what document processing image belongs.
  176.  
  177.        '''
  178.         self.debug('document_handler on {} started'.format(task[0]))
  179.         sleep(randint(0, 4))
  180.         document = dict(content=task[0], images=[], images_cnt=None, feed_id=task[1], category=task[2])
  181.         self.results.append(document)
  182.         possible_images = randint(0, 50)
  183.         # represents possible number of images in documents that needs to be checked
  184.         document['images_cnt'] = possible_images
  185.         for i in range(possible_images):
  186.             self.queue.put((0, 'image_handler', uuid.uuid4(), document))
  187.         self.flush_results()
  188.         self.debug('document_handler {} completed'.format(task[0]))
  189.  
  190.     @with_timeout(2, register_timeout_event)
  191.     def feed_handler(self, *task):
  192.         '''Complete docs at real implementation!
  193.        Parse RSS feed, nextly find new items and pass them to documents handler
  194.  
  195.        '''
  196.         self.debug('feed_handler on {} started'.format(task[0]))
  197.         sleep(randint(0, 3))
  198.         for i in range(randint(1, 10)):
  199.             self.queue.put((1, 'document_handler', uuid.uuid4(), task[0], 'root'))
  200.         self.debug('feed_handler {} completed'.format(task[0]))
  201.  
  202. if __name__ == '__main__':
  203.     tasks = [(2, 'feed_handler', uuid.uuid4()) for i in xrange(5)]
  204.     log = logging.getLogger()
  205.     handler = logging.StreamHandler()
  206.     formatter = logging.Formatter("%(asctime)s -> %(message)s")
  207.     handler.setFormatter(formatter)
  208.     log.addHandler(handler)
  209.     log.setLevel(logging.DEBUG)
  210.     processing = RssProcessingPool(size=100, logger=log)
  211.     processing.dispatch(tasks)
  212.     print processing.get_stats()
Advertisement
Add Comment
Please, Sign In to add comment