Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import subprocess
- import os
- import time
- files = <list of file names>
- command = "/bin/touch"
- processes = set()
- max_processes = 5
- for name in files:
- processes.add(subprocess.Popen([command, name]))
- if len(processes) >= max_processes:
- os.wait()
- processes.difference_update([
- p for p in processes if p.poll() is not None])
- for name in files:
- processes.add(subprocess.Popen([command, name]))
- while len(processes) >= max_processes:
- time.sleep(.1)
- processes.difference_update([
- p for p in processes if p.poll() is not None])
- import subprocess
- import os
- import time
- files = <list of file names>
- command = "/bin/touch"
- processes = set()
- max_processes = 5
- for name in files:
- processes.add(subprocess.Popen([command, name]))
- if len(processes) >= max_processes:
- os.wait()
- processes.difference_update(
- [p for p in processes if p.poll() is not None])
- #Check if all the child processes were closed
- for p in processes:
- if p.poll() is None:
- p.wait()
- #!/usr/bin/python
- import threading
- import os
- semaphore = threading.Semaphore(4)
- def run_command(cmd):
- semaphore.acquire()
- try:
- os.system(cmd)
- finally:
- semaphore.release()
- def run_command(cmd):
- with semaphore:
- os.system(cmd)
- for i in range(8):
- threading.Thread(target=run_command, args=("sleep 10", )).start()
- aw@aw-laptop:~/personal/stackoverflow$ time python 4992400.py
- real 0m20.032s
- user 0m0.020s
- sys 0m0.008s
- def removeFinishedProcesses(processes):
- """ given a list of (commandString, process),
- remove those that have completed and return the result
- """
- newProcs = []
- for pollCmd, pollProc in processes:
- retCode = pollProc.poll()
- if retCode==None:
- # still running
- newProcs.append((pollCmd, pollProc))
- elif retCode!=0:
- # failed
- raise Exception("Command %s failed" % pollCmd)
- else:
- logging.info("Command %s completed successfully" % pollCmd)
- return newProcs
- def runCommands(commands, maxCpu):
- processes = []
- for command in commands:
- logging.info("Starting process %s" % command)
- proc = subprocess.Popen(shlex.split(command))
- procTuple = (command, proc)
- processes.append(procTuple)
- while len(processes) >= maxCpu:
- time.sleep(.2)
- processes = removeFinishedProcesses(processes)
- # wait for all processes
- while len(processes)>0:
- time.sleep(0.5)
- processes = removeFinishedProcesses(processes)
- logging.info("All processes completed")
- from subprocess import PIPE
- import subprocess
- import time
- def submit_job_max_len(job_list, max_processes):
- sleep_time = 0.1
- processes = list()
- for command in job_list:
- print 'running {n} processes. Submitting {proc}.'.format(n=len(processes),
- proc=str(command))
- processes.append(subprocess.Popen(command, shell=False, stdout=None,
- stdin=PIPE))
- while len(processes) >= max_processes:
- time.sleep(sleep_time)
- processes = [proc for proc in processes if proc.poll() is None]
- while len(processes) > 0:
- time.sleep(sleep_time)
- processes = [proc for proc in processes if proc.poll() is None]
- cmd = '/bin/bash run_what.sh {n}'
- job_list = ((cmd.format(n=i)).split() for i in range(100))
- submit_job_max_len(job_list, max_processes=50)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement