Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/python
- # coding=utf8
- # vim: sw=4 ts=4 expandtab ai
- import sys
- import logging
- from random import randint
- from gevent import pool, queue, sleep
- LOG = logging.getLogger()
- MAX_WORKERS = 10
- def do_first(task_id, wqueue):
- LOG.debug("do_first %d", task_id)
- # added second step tasks to the same queue
- for i in range(3):
- wqueue.put((1, "second_task", "%d-%d" % (task_id, i)))
- LOG.debug("created 10 tasks %s-*" % task_id)
- sleep(10)
- LOG.debug("do_first %d done" % task_id)
- def do_second(task_id):
- LOG.debug("do_second %s", task_id)
- sleep(randint(1, 2))
- LOG.debug("do_second %s done", task_id)
- def do_it(max_workers=MAX_WORKERS):
- # add first step tasks to the queue
- wqueue = queue.PriorityQueue()
- for i in range(10):
- wqueue.put((2, 'first_task', i, wqueue))
- workers = {"first_task": do_first, "second_task": do_second}
- wpool = pool.Pool(max_workers)
- # dispatch tasks
- while True:
- while wpool.free_count():
- try:
- task = wqueue.get_nowait()
- except queue.Empty:
- break
- LOG.debug("got from the queue: %s", task[:3])
- LOG.debug("do_it: starting task %s" % task[1])
- wpool.spawn(workers[task[1]], *(task[2:]))
- LOG.debug("do_it: task %s has been started" % task[1])
- if wpool.free_count() == max_workers:
- break
- sleep(1)
- return 0
- def main(argv):
- # configuring logging
- handler = logging.StreamHandler()
- formatter = logging.Formatter("%(asctime)s -> %(message)s")
- handler.setFormatter(formatter)
- LOG.addHandler(handler)
- LOG.setLevel(logging.DEBUG)
- if len(argv) == 1:
- max_workers = MAX_WORKERS
- else:
- max_workers = int(argv[1])
- return do_it(max_workers)
- if __name__ == "__main__":
- sys.exit(main(sys.argv))
Advertisement
Add Comment
Please, Sign In to add comment