Guest User

threading vs multiprocessing

a guest
Nov 27th, 2014
252
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 2.67 KB | None | 0 0
  1. from __future__ import print_function
  2. import threading
  3. import mmap
  4. import multiprocessing as mp
  5.  
  6. try:
  7.    xrange
  8. except NameError:
  9.    xrange = range
  10.  
  11. class ThreadOrProcess(object):
  12.  
  13.     def __init__(self, data, use_threading=True):
  14.         self.data = data
  15.         self.use_threading = use_threading
  16.  
  17.         if self.use_threading:
  18.             self.lookup = dict()
  19.             self.thread = threading.Thread(target=self.threaded_create_lookup,
  20.                                            name="lookup maker thread")
  21.             self.thread.start()
  22.             #self.check_progress()
  23.             self.thread.join()
  24.  
  25.         else:
  26.             self.lookup = mp.Manager().dict()
  27.             self.process = mp.Process(target=self.mp_create_lookup,
  28.                                       name="lookup maker process")
  29.             self.process.start()
  30.             #self.check_progress()
  31.             self.process.join()
  32.  
  33.     def check_progress(self):
  34.         before, after = float("nan"), float("nan")
  35.         while before != after:
  36.             before = len(self.lookup)
  37.             print("%s: %d items" % (time.asctime(), before))
  38.             time.sleep(0.01)
  39.             after = len(self.lookup)
  40.  
  41.     def threaded_create_lookup(self):
  42.         lino, record_start = 0, 0
  43.         for line in self.data:
  44.             if not line:
  45.                 break
  46.             self.lookup[lino] = record_start
  47.             lino += 1
  48.             record_start += len(line)
  49.         return self.lookup
  50.  
  51.     def mp_create_lookup(self):
  52.         lino, record_start = 0, 0
  53.         for line in self.data:
  54.             if not line:
  55.                 break
  56.             self.lookup[lino] = record_start
  57.             lino += 1
  58.             record_start += len(line)
  59.         return self.lookup
  60.  
  61. if __name__ == "__main__":
  62.     import os
  63.     import csv
  64.     import time
  65.     import string
  66.  
  67.     # ---- create test data
  68.     if not os.path.exists("testfile.csv"):
  69.         with open("testfile.csv", "wb") as f:
  70.             writer = csv.writer(f)
  71.             record = list(string.ascii_letters)
  72.             for i in xrange(10 ** 5):
  73.                 writer.writerow([i] + record)
  74.  
  75.     # ---- threading
  76.     start = time.time()
  77.     f = open("testfile.csv", "r")
  78.     #threader = ThreadOrProcess(f, True)
  79.     #print(len(dict(threader.lookup)))
  80.     print("threading approach took: %3.3f seconds" % (time.time() - start))
  81.  
  82.     print("***********************************************")  
  83.     # ---- multiprocessing
  84.     start = time.time()
  85.     f.seek(0)
  86.     processer = ThreadOrProcess(f, False)
  87.     print(len(dict(processer.lookup)))
  88.     print("mp approach took: %3.3f seconds" % (time.time() - start))
Advertisement
Add Comment
Please, Sign In to add comment