Advertisement
Guest User

Untitled

a guest
Aug 5th, 2015
136
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. from Queue import PriorityQueue
  2. from threading import Thread
  3. from time import sleep
  4. from random import randrange
  5. import multiprocessing
  6.  
  7. class MyPriorityQueue(PriorityQueue):
  8.     def __init__(self):
  9.         PriorityQueue.__init__(self)
  10.         self.counter = 0
  11.  
  12.     def put(self, item, priority):
  13.         PriorityQueue.put(self, (priority, self.counter, item))
  14.         self.counter += 1
  15.  
  16.     def get(self, *args, **kwargs):
  17.         _, _, item = PriorityQueue.get(self, *args, **kwargs)
  18.         return item
  19.  
  20. class Worker(Thread):
  21.     """Thread executing tasks from a given tasks queue"""
  22.     def __init__(self, tasks):
  23.         Thread.__init__(self)
  24.         self.tasks = tasks
  25.         self.daemon = True
  26.         self.start()
  27.  
  28.     def run(self):
  29.         while True:
  30.             # self.tasks = PriorityQueue
  31.             # .get -> function, arg, kargs
  32.             # dosmth(interval, foo=bar, zeugs=toll)
  33.             func, args, kargs = self.tasks.get()
  34.             try:
  35.                 func(*args, **kargs)
  36.             except Exception, e:
  37.                 print e
  38.             finally:
  39.                 self.tasks.task_done()
  40.  
  41. class ThreadPool:
  42.     """Pool of threads consuming tasks from a queue"""
  43.     def __init__(self, num_threads):
  44.         self.tasks = MyPriorityQueue()
  45.         for _ in range(num_threads): Worker(self.tasks)
  46.  
  47.     def add_task(self, priority, func, *args, **kargs):
  48.         """Add a task to the queue"""
  49.         self.tasks.put((func, args, kargs), priority)
  50.  
  51.     def wait_completion(self):
  52.         """Wait for completion of all the tasks in the queue"""
  53.         self.tasks.join()
  54.  
  55. def waitAndSleep(time,i):
  56.     if i % 20 == 0:
  57.         pool.add_task(-10, waitAndSleep, d, i+101)
  58.     print 'sleeping for (%d)sec at %d' % (time,i)
  59.     sleep(time)
  60.  
  61.  
  62.  
  63. if __name__ == '__main__':
  64.  
  65.     delays = [randrange(1, 4) for i in range(100)]
  66.  
  67.     cpus = multiprocessing.cpu_count()
  68.     print cpus
  69.     pool = ThreadPool(cpus)
  70.  
  71.     while True:
  72.         task = fetchtaskFromBoinc()
  73.         pool.add_task(0,task)
  74.  
  75.     for i, d in enumerate(delays):
  76.         pool.add_task(0, waitAndSleep, d, i)
  77.  
  78.     pool.wait_completion()%
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement