Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import gevent
- import logging
- import time
- import uuid
- from random import randint
- from gevent import queue, pool, monkey, sleep
- class with_timeout(object):
- def __init__(self, timeout, callback=None):
- '''Load timeout settings to decorator
- timeout -- seconds value
- callback -- if timeout reached and callback is set, will be fired with function arguments
- exception -- action to be taken if decorated function running to long, options are following:
- + False -> will supress any error
- + None -> will raise gevent.Timeout exception
- + CustomError -> will raise CustomError exception
- Usage:
- @Timeout(5) # 5 seconds to run this task
- def foo(*args, **kwargs):
- ...
- '''
- self.timeout = timeout
- self.callback = callback
- def __call__(self, f):
- '''Decorate requested function'''
- def wrapped(*args):
- try:
- with gevent.Timeout(self.timeout, None):
- return f(*args)
- except gevent.Timeout:
- if self.callback is not None:
- return self.callback(*args, **{'func': f.__name__})
- return wrapped
- class DispatchablePoolError(Exception): pass
- class HandlerError(DispatchablePoolError): pass
- class HandlerMissingError(HandlerError): pass
- class HandlerTimeoutError(HandlerError): pass
- class DispatchablePool(pool.Pool):
- '''This is gevent.Pool implementation with concurence task dispatch functionality.
- You can subclass this class, build your own logic and then simply
- call to instance.dispatch(tasks) in order to process your tasks asynchronously
- '''
- def __init__(self, size=1, handlers=None, logger=None, apply_patch=True):
- '''Pool initialization:
- size -- how much asynchronicity you need?
- handlers -- dictionary of custom task handlers, these handlers
- has higher priority than thoose defined in class object itself.
- Key of dictionary element maps to task name.
- logger -- logging object, None value disables this functionality.
- '''
- self.handlers = handlers
- if handlers is None:
- self.handlers = {}
- self.size = size
- self.queue = queue.PriorityQueue()
- self.logger = logger
- if apply_patch:
- monkey.patch_all()
- self.start_time = 0
- self.end_time = 0
- self.tasks_done = False
- pool.Pool.__init__(self, size=self.size)
- def debug(self, message):
- '''Log message via logger object'''
- if self.logger:
- self.logger.debug(message)
- def get_stats(self):
- '''Current status of jobs'''
- if self.tasks_done:
- return "Jobs done in {:.3}s".format(self.end_time - self.start_time)
- return "Jobs are not finished yet"
- def dispatch(self, tasks):
- '''Run defined asynchronous tasks in an endless loop
- tasks -- iterator object of tasks.
- Single task typically looks like (priority, task, ...)
- '''
- self.start_time = time.time()
- for _task in tasks:
- self.queue.put(_task)
- while True:
- while self.free_count():
- try:
- task = self.queue.get_nowait()
- except queue.Empty:
- break
- # self.debug("Got task from queue: {}".format(task[:3]))
- handler = getattr(self, task[1], None)
- if task[1] in self.handlers:
- handler = self.handlers[task[1]]
- if handler is None:
- raise HandlerMissingError, \
- "Required handler for task {} not defined".format(task[1])
- self.spawn(handler, *(task[2:]))
- # self.debug("Started task: {}".format(task[1]))
- if self.free_count() == self.size:
- self.end_time = time.time()
- self.tasks_done = True
- break
- sleep(1)
- def image_handler_failed(*task, **kwargs):
- '''Update images count awaiting processing at current document
- Due to timeout we should decrease its number by one
- '''
- document = task[2]
- document['images_cnt'] -= 1
- processing.flush_results()
- register_timeout_event(*task, **kwargs)
- def register_timeout_event(*task, **kwargs):
- '''General callback for timed out handlers'''
- pool = task[0]
- pool.debug("{} task timed out at {}".format(kwargs.get('func'), task[1]))
- class RssProcessingPool(DispatchablePool):
- # TODO: Make use of gevent.Timeout in content handlers, I'm afraid I can't use Timeout decorator :(
- '''Use DisplatchablePool to process multiple RSS feeds in async mode.
- Results are flushed to storage as soon as ready.
- '''
- def __init__(self, **kwargs):
- '''Setup pool
- **kwargs -- same params as at superclass
- '''
- self.results = []
- DispatchablePool.__init__(self, **kwargs)
- def flush_results(self):
- '''Save ready documents to db, free up memory
- Ready document does not have any images scheduled to process.
- '''
- for i, doc in enumerate(self.results):
- if not doc['images_cnt']:
- self.debug("Document {} saved on disk".format(doc['content']))
- del self.results[i]
- @with_timeout(1, image_handler_failed)
- def image_handler(self, *task):
- '''Complete docs at real implementation!
- Process image. This handler *knows* about document its working with, thats very usefull for consistency
- of document object.
- '''
- self.debug('image_handler on {} started'.format(task[0]))
- document = task[1]
- document['images'].append(task[0])
- sleep(randint(0, 5))
- # decrease number of possible images for this documents that's are awaiting to check
- document['images_cnt'] -= 1
- self.flush_results()
- self.debug('image_handler on {} completed'.format(task[0]))
- @with_timeout(2, register_timeout_event)
- def document_handler(self, *task):
- '''Complete docs at real implementation
- Document Handler used to parse single documents from rss feeds, this method
- analyzes document content and *could* run additional image rertival task.
- Something interesting happens here - we setup document and then we pass it to image hadler,
- so image handler will *know* what document processing image belongs.
- '''
- self.debug('document_handler on {} started'.format(task[0]))
- sleep(randint(0, 4))
- document = dict(content=task[0], images=[], images_cnt=None, feed_id=task[1], category=task[2])
- self.results.append(document)
- possible_images = randint(0, 50)
- # represents possible number of images in documents that needs to be checked
- document['images_cnt'] = possible_images
- for i in range(possible_images):
- self.queue.put((0, 'image_handler', uuid.uuid4(), document))
- self.flush_results()
- self.debug('document_handler {} completed'.format(task[0]))
- @with_timeout(2, register_timeout_event)
- def feed_handler(self, *task):
- '''Complete docs at real implementation!
- Parse RSS feed, nextly find new items and pass them to documents handler
- '''
- self.debug('feed_handler on {} started'.format(task[0]))
- sleep(randint(0, 3))
- for i in range(randint(1, 10)):
- self.queue.put((1, 'document_handler', uuid.uuid4(), task[0], 'root'))
- self.debug('feed_handler {} completed'.format(task[0]))
- if __name__ == '__main__':
- tasks = [(2, 'feed_handler', uuid.uuid4()) for i in xrange(5)]
- log = logging.getLogger()
- handler = logging.StreamHandler()
- formatter = logging.Formatter("%(asctime)s -> %(message)s")
- handler.setFormatter(formatter)
- log.addHandler(handler)
- log.setLevel(logging.DEBUG)
- processing = RssProcessingPool(size=100, logger=log)
- processing.dispatch(tasks)
- print processing.get_stats()
Advertisement
Add Comment
Please, Sign In to add comment