Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python
- #
- # This example shows how to launch multiple processes in parallel on a single machine with multiple cpus
- # There is no communication between processes and no data are gathered in the end.
- # Each process executes the same function but with different arguments.
- # The script wait for all sub-processes to be done, then execute another task.
- #
- # How to run on your computer:
- # python multiprocessing_eg_02.py
- #
- # How to run on datarmor:
- # qsub -q sequentiel -N test_pythonpar launch_this.sh
- #
- # The launcher script is: launch_this.sh with:
- # ======================================================================
- # #!/bin/bash
- # source ~/.bashrc
- # source activate obidam
- # python /home1/datahome/gmaze/work/Play_Ground/python/parallel/examples/multiprocessing_eg_02datarmor.py
- # exit 0
- # ======================================================================
- #
- # Eg:
- # >> qsub -q sequentiel -N test_python multiprocessing_eg_02.py
- # >> qstat -u gmaze
- #
- # datarmor0:
- # Req'd Req'd Elap
- # Job ID Username Queue Jobname SessID NDS TSK Memory Time S Time
- # --------------- -------- -------- ---------- ------ --- --- ------ ----- - -----
- # 508731.datarmor gmaze ice_1t test_pytho 23442 1 1 500mb 00:05 R 00:00
- # r2i2n10/4
- #
- # >> more test_pythonpar.o508731
- # ======================================================================
- # multiprocessing_eg_02.py.
- # This machine has 56 cores.
- # ======================================================================
- #
- # - Process named: dummy_name_1 (running on r2i2n10) just started.
- # It received argument:
- # 61
- #
- # - Process named: dummy_name_2 (running on r2i2n10) just started.
- # It received argument:
- # 73
- #
- # - Process named: dummy_name_7 (running on r2i2n10) just started.
- # It received argument:
- # 33
- # [...]
- #
- # End of the main program (this text will be displayed after all processes finished)
- #
- # Created: 2013-02-06.
- # Copyright (c) 2013, Guillaume Maze (Ifremer, Laboratoire de Physique des Oceans).
- # Revision by G. Maze on 2017-11-23: Added help for datarmor
- # All rights reserved.
- import sys, socket, time
- import multiprocessing
- import numpy as np
- def worker(num):
- """
- A worker function: defines the task to be performed by a process.
- This function is executed by each process with a
- different argument 'num'
- """
- # Retrieve the name of this process:
- procname = multiprocessing.current_process().name
- # Retrive the name of the machine:
- hostname = socket.gethostname()
- # Print out some usefull info:
- msg = "\t- Process named: %s (running on %s) just started.\n\tIt received argument:\n\t%i\n\n"
- sys.stdout.write(msg % (procname,hostname,num))
- # Dummy action, simple wait for anywhere between 4 and 5 seconds
- time.sleep(np.random.randint(4,5,1))
- if __name__ == '__main__':
- # How many core have this machine ?
- N_CORES = multiprocessing.cpu_count()
- print ('%s\nmultiprocessing_eg_02.py.\nThis machine has %i cores.\n%s\n')%("="*70,N_CORES,"="*70)
- # Init the list which is going to hold the list of processes:
- process_list = []
- # Create a dummy list of parameters.
- param_list = np.random.randint(20,100,N_CORES)
- # Now we will start N_CORES processes.
- # All will run the 'worker' function with a dummy parameter from 'param_list'
- for i in range(N_CORES):
- # Define a process:
- p = multiprocessing.Process(target=worker, args=(param_list[i],), name=('dummy_name_%i')%(i))
- # Start the process:
- p.start()
- # Register the process in a list
- process_list.append(p)
- # Now we tell the script to wait for all processes to finish before moving on
- for p in process_list:
- p.join()
- # This won't be executed before the end of all processes:
- print "End of the main program (this text will be displayed after all processes finished)\n"
Add Comment
Please, Sign In to add comment