SHARE
TWEET

Untitled

a guest Jul 20th, 2019 78 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import subprocess
  2. import os
  3. import time
  4.  
  5. files = <list of file names>
  6. command = "/bin/touch"
  7. processes = set()
  8. max_processes = 5
  9.  
  10. for name in files:
  11.     processes.add(subprocess.Popen([command, name]))
  12.     if len(processes) >= max_processes:
  13.         os.wait()
  14.         processes.difference_update([
  15.             p for p in processes if p.poll() is not None])
  16.      
  17. for name in files:
  18.     processes.add(subprocess.Popen([command, name]))
  19.     while len(processes) >= max_processes:
  20.         time.sleep(.1)
  21.         processes.difference_update([
  22.             p for p in processes if p.poll() is not None])
  23.      
  24. import subprocess
  25. import os
  26. import time
  27.  
  28. files = <list of file names>
  29. command = "/bin/touch"
  30. processes = set()
  31. max_processes = 5
  32.  
  33. for name in files:
  34.     processes.add(subprocess.Popen([command, name]))
  35.     if len(processes) >= max_processes:
  36.         os.wait()
  37.         processes.difference_update(
  38.             [p for p in processes if p.poll() is not None])
  39. #Check if all the child processes were closed
  40. for p in processes:
  41.     if p.poll() is None:
  42.         p.wait()
  43.      
  44. #!/usr/bin/python
  45.  
  46. import threading
  47. import os
  48.      
  49. semaphore = threading.Semaphore(4)
  50.      
  51. def run_command(cmd):
  52.     semaphore.acquire()
  53.     try:
  54.         os.system(cmd)
  55.     finally:
  56.         semaphore.release()
  57.      
  58. def run_command(cmd):
  59.     with semaphore:
  60.         os.system(cmd)
  61.      
  62. for i in range(8):
  63.     threading.Thread(target=run_command, args=("sleep 10", )).start()
  64.      
  65. aw@aw-laptop:~/personal/stackoverflow$ time python 4992400.py
  66.  
  67. real    0m20.032s                                                                                                                                                                  
  68. user    0m0.020s                                                                                                                                                                    
  69. sys     0m0.008s
  70.      
  71. def removeFinishedProcesses(processes):
  72.     """ given a list of (commandString, process),
  73.         remove those that have completed and return the result
  74.     """
  75.     newProcs = []
  76.     for pollCmd, pollProc in processes:
  77.         retCode = pollProc.poll()
  78.         if retCode==None:
  79.             # still running
  80.             newProcs.append((pollCmd, pollProc))
  81.         elif retCode!=0:
  82.             # failed
  83.             raise Exception("Command %s failed" % pollCmd)
  84.         else:
  85.             logging.info("Command %s completed successfully" % pollCmd)
  86.     return newProcs
  87.  
  88. def runCommands(commands, maxCpu):
  89.             processes = []
  90.             for command in commands:
  91.                 logging.info("Starting process %s" % command)
  92.                 proc =  subprocess.Popen(shlex.split(command))
  93.                 procTuple = (command, proc)
  94.                 processes.append(procTuple)
  95.                 while len(processes) >= maxCpu:
  96.                     time.sleep(.2)
  97.                     processes = removeFinishedProcesses(processes)
  98.  
  99.             # wait for all processes
  100.             while len(processes)>0:
  101.                 time.sleep(0.5)
  102.                 processes = removeFinishedProcesses(processes)
  103.             logging.info("All processes completed")
  104.      
  105. from subprocess import PIPE
  106. import subprocess
  107. import time
  108.  
  109.  
  110. def submit_job_max_len(job_list, max_processes):
  111.   sleep_time = 0.1
  112.   processes = list()
  113.   for command in job_list:
  114.     print 'running {n} processes. Submitting {proc}.'.format(n=len(processes),
  115.         proc=str(command))
  116.     processes.append(subprocess.Popen(command, shell=False, stdout=None,
  117.       stdin=PIPE))
  118.     while len(processes) >= max_processes:
  119.       time.sleep(sleep_time)
  120.       processes = [proc for proc in processes if proc.poll() is None]
  121.   while len(processes) > 0:
  122.     time.sleep(sleep_time)
  123.     processes = [proc for proc in processes if proc.poll() is None]
  124.  
  125.  
  126. cmd = '/bin/bash run_what.sh {n}'
  127. job_list = ((cmd.format(n=i)).split() for i in range(100))
  128. submit_job_max_len(job_list, max_processes=50)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!
 
Top