Advertisement
joxeankoret

Python script to maintain a list of N number of workers

Dec 6th, 2013
145
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.39 KB | None | 0 0
  1. #!/usr/bin/python
  2. # -*- coding: utf-8 -*-
  3.  
  4. import os
  5. import sys
  6. import time
  7.  
  8. from multiprocessing import Process, cpu_count
  9.  
  10. #-----------------------------------------------------------------------
  11. def log(msg):
  12.   print "[%s %d] %s" % (time.asctime(), os.getpid(), msg)
  13.  
  14. #-----------------------------------------------------------------------
  15. def dummy_process():
  16.   import random
  17.   t = random.randint(0, 2)
  18.   log("Sleeping %d second(s)..." % t)
  19.   time.sleep(t)
  20.   log("Done, bye")
  21.  
  22. #-----------------------------------------------------------------------
  23. def process_manager(total_procs, target, wait_time=0.2):
  24.   """ Always maintain a total of @total_procs running @target and
  25.      waiting for each thread to finish @wait_time second(s). """
  26.   procs = []
  27.   log("Maximum number of processes in pool is %d" % total_procs)
  28.   while 1:
  29.     if len(procs) < total_procs:
  30.       log("Starting process %d" % (len(procs)+1))
  31.       p = Process(target=target)
  32.       p.start()
  33.       procs.append(p)
  34.       log("Total of %d process(es) started" % len(procs))
  35.     else:
  36.       i = 0
  37.       for p in list(procs):
  38.         p.join(wait_time)
  39.         if not p.is_alive():
  40.           log("Process finished, deleting and starting a new one...")
  41.           del procs[i]
  42.           continue
  43.         i += 1
  44.  
  45. if __name__ == "__main__":
  46.   total_procs = cpu_count()*5
  47.   manage_pool(total_procs, dummy_process)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement