daily pastebin goal
29%
SHARE
TWEET

Untitled

a guest May 16th, 2018 93 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import numpy as np
  2. from multiprocessing import Process, Manager, Value, Array
  3. import time
  4. import humanize
  5.  
  6.  
  7. #actual computation
  8. def f(a):
  9.     return a-1.0;
  10. # obs: with such a simple computation, there will advantage will be no computation time advantage.
  11. # use this for hardcore stuff that deals with peripherals such as a disk read
  12.  
  13. #computation wrapper
  14. def process_target(pidx, Global, res, count, N, P):
  15.  
  16.     for i in range(pidx, N, P):
  17.        
  18.         with count.get_lock():
  19.             count.value += 1;
  20.            
  21.        
  22.         res[i] =  f(Global.mat[i])
  23.        
  24.         if Global.run is False:
  25.             break;
  26.  
  27.  
  28. if __name__ == '__main__':
  29.  
  30.     manager = Manager()  
  31.     Global = manager.Namespace()
  32.    
  33.    
  34.     N = 100000;                      #job size
  35.    
  36.     P = 12;                          #process number
  37.                  
  38.     Global.mat = np.random.rand(N);  #input vector
  39.    
  40.     res = Array('d', N, lock = False) #output vector
  41.    
  42.     count = Value('i', 0)             #counter
  43.    
  44.     Global.run = True;                #used for greceful halt
  45.    
  46.    
  47.     #init procecess
  48.     proc_list = [Process(target=process_target, args=(i, Global, res, count, N, P)) for i in range(P)]
  49.    
  50.     #start procecess
  51.     [p.start() for p in proc_list]
  52.    
  53.    
  54.     ts = time.time()
  55.     elapsed = 0;
  56.     c = 0;
  57.     while any([p.is_alive() for p in proc_list]):
  58.         try:
  59.             time.sleep(2)
  60.             elapsed_old = elapsed;
  61.             elapsed = time.time();
  62.            
  63.             c_old = c
  64.             c = count.value
  65.            
  66.             remaining = (elapsed - elapsed_old)*(N  - c)/(c + -c_old)
  67.            
  68.             print(humanize.naturaltime(remaining, future=True));
  69.  
  70.         except KeyboardInterrupt:
  71.             print("KeyboardInterrupt")
  72.             Global.run = False;
  73.             [p.terminate() for p in proc_list]
  74.             break;
  75.         except ZeroDivisionError:
  76.             time.sleep(1)
  77.  
  78.  
  79.    
  80.     [p.join() for p in proc_list]
  81.        
  82.     print("Final time: " + humanize.naturaldelta(time.time() - ts))
  83.        
  84.    
  85.     print("{} out of {} elements were processed".format(count.value, N))
  86.        
  87.     #get results
  88.     res_npy =  np.frombuffer(res, dtype='d');
RAW Paste Data
Top