daily pastebin goal
83%
SHARE
TWEET

Untitled

a guest May 16th, 2018 94 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import numpy as np
  2. from multiprocessing import Process, Manager, Value, Array
  3. import time
  4. import humanize
  5.  
  6.  
  7. #actual computation
  8. def f(a):
  9.     return a-1.0;
  10. # obs: with such a simple computation, there will advantage will be no computation time advantage.
  11. # use this for hardcore stuff that deals with peripherals such as a disk read
  12.  
  13. #computation wrapper
  14. def process_target(pidx, Global, res, count, N, P):
  15.  
  16.     for i in range(pidx, N, P):
  17.        
  18.         with count.get_lock():
  19.             count.value += 1;
  20.            
  21.        
  22.         res[i] =  f(Global.mat[i])
  23.        
  24.         if Global.run is False:
  25.             break;
  26.  
  27.  
  28. if __name__ == '__main__':
  29.  
  30.     manager = Manager()  
  31.     Global = manager.Namespace()
  32.    
  33.    
  34.     N = 100000;                      #job size
  35.    
  36.     P = 12;                          #process number
  37.                  
  38.     Global.mat = np.random.rand(N);  #input vector
  39.    
  40.     res = Array('d', N, lock = False) #output vector
  41.    
  42.     count = Value('i', 0)             #counter
  43.    
  44.     Global.run = True;                #used for greceful halt
  45.    
  46.    
  47.     #init procecess
  48.     proc_list = [Process(target=process_target, args=(i, Global, res, count, N, P)) for i in range(P)]
  49.    
  50.     #start procecess
  51.     [p.start() for p in proc_list]
  52.    
  53.    
  54.     ts = time.time()
  55.     elapsed = 0;
  56.     c = 0;
  57.     while any([p.is_alive() for p in proc_list]):
  58.         try:
  59.             time.sleep(2)
  60.             elapsed_old = elapsed;
  61.             elapsed = time.time();
  62.            
  63.             c_old = c
  64.             c = count.value
  65.            
  66.             remaining = (elapsed - elapsed_old)*(N  - c)/(c + -c_old)
  67.            
  68.             print(humanize.naturaltime(remaining, future=True));
  69.  
  70.         except KeyboardInterrupt:
  71.             print("KeyboardInterrupt")
  72.             Global.run = False;
  73.             [p.terminate() for p in proc_list]
  74.             break;
  75.         except ZeroDivisionError:
  76.             time.sleep(1)
  77.  
  78.  
  79.    
  80.     [p.join() for p in proc_list]
  81.        
  82.     print("Final time: " + humanize.naturaldelta(time.time() - ts))
  83.        
  84.    
  85.     print("{} out of {} elements were processed".format(count.value, N))
  86.        
  87.     #get results
  88.     res_npy =  np.frombuffer(res, dtype='d');
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top