Guest User

Gevent task dispatching example

a guest
Dec 7th, 2010
319
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.91 KB | None | 0 0
  1. #!/usr/bin/python
  2. # coding=utf8
  3. # vim: sw=4 ts=4 expandtab ai
  4.  
  5. import sys
  6. import logging
  7. from random import randint
  8.  
  9. from gevent import pool, queue, sleep
  10.  
  11. LOG = logging.getLogger()
  12.  
  13. MAX_WORKERS = 10
  14.  
  15. def do_first(task_id, wqueue):
  16.     LOG.debug("do_first %d", task_id)
  17.     # added second step tasks to the same queue
  18.     for i in range(3):
  19.         wqueue.put((1, "second_task", "%d-%d" % (task_id, i)))
  20.     LOG.debug("created 10 tasks %s-*" % task_id)
  21.     sleep(10)
  22.     LOG.debug("do_first %d done" % task_id)
  23.  
  24. def do_second(task_id):
  25.     LOG.debug("do_second %s", task_id)
  26.     sleep(randint(1, 2))
  27.     LOG.debug("do_second %s done", task_id)
  28.  
  29. def do_it(max_workers=MAX_WORKERS):
  30.  
  31.     # add first step tasks to the queue
  32.     wqueue = queue.PriorityQueue()
  33.     for i in range(10):
  34.         wqueue.put((2, 'first_task', i, wqueue))
  35.  
  36.     workers = {"first_task": do_first, "second_task": do_second}
  37.     wpool = pool.Pool(max_workers)
  38.  
  39.     # dispatch tasks
  40.     while True:
  41.         while wpool.free_count():
  42.             try:
  43.                 task = wqueue.get_nowait()
  44.             except queue.Empty:
  45.                 break
  46.             LOG.debug("got from the queue: %s", task[:3])
  47.             LOG.debug("do_it: starting task %s" % task[1])
  48.             wpool.spawn(workers[task[1]], *(task[2:]))
  49.             LOG.debug("do_it: task %s has been started" % task[1])
  50.  
  51.         if wpool.free_count() == max_workers:
  52.             break
  53.  
  54.         sleep(1)
  55.  
  56.     return 0
  57.  
  58. def main(argv):
  59.  
  60.     # configuring logging
  61.     handler = logging.StreamHandler()
  62.     formatter = logging.Formatter("%(asctime)s -> %(message)s")
  63.     handler.setFormatter(formatter)
  64.     LOG.addHandler(handler)
  65.     LOG.setLevel(logging.DEBUG)
  66.  
  67.     if len(argv) == 1:
  68.         max_workers = MAX_WORKERS
  69.     else:
  70.         max_workers = int(argv[1])
  71.  
  72.     return do_it(max_workers)
  73.  
  74. if __name__ == "__main__":
  75.     sys.exit(main(sys.argv))
Advertisement
Add Comment
Please, Sign In to add comment