Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from __future__ import print_function
- import threading
- import mmap
- import multiprocessing as mp
- try:
- xrange
- except NameError:
- xrange = range
- class ThreadOrProcess(object):
- def __init__(self, data, use_threading=True):
- self.data = data
- self.use_threading = use_threading
- if self.use_threading:
- self.lookup = dict()
- self.thread = threading.Thread(target=self.threaded_create_lookup,
- name="lookup maker thread")
- self.thread.start()
- #self.check_progress()
- self.thread.join()
- else:
- self.lookup = mp.Manager().dict()
- self.process = mp.Process(target=self.mp_create_lookup,
- name="lookup maker process")
- self.process.start()
- #self.check_progress()
- self.process.join()
- def check_progress(self):
- before, after = float("nan"), float("nan")
- while before != after:
- before = len(self.lookup)
- print("%s: %d items" % (time.asctime(), before))
- time.sleep(0.01)
- after = len(self.lookup)
- def threaded_create_lookup(self):
- lino, record_start = 0, 0
- for line in self.data:
- if not line:
- break
- self.lookup[lino] = record_start
- lino += 1
- record_start += len(line)
- return self.lookup
- def mp_create_lookup(self):
- lino, record_start = 0, 0
- for line in self.data:
- if not line:
- break
- self.lookup[lino] = record_start
- lino += 1
- record_start += len(line)
- return self.lookup
- if __name__ == "__main__":
- import os
- import csv
- import time
- import string
- # ---- create test data
- if not os.path.exists("testfile.csv"):
- with open("testfile.csv", "wb") as f:
- writer = csv.writer(f)
- record = list(string.ascii_letters)
- for i in xrange(10 ** 5):
- writer.writerow([i] + record)
- # ---- threading
- start = time.time()
- f = open("testfile.csv", "r")
- #threader = ThreadOrProcess(f, True)
- #print(len(dict(threader.lookup)))
- print("threading approach took: %3.3f seconds" % (time.time() - start))
- print("***********************************************")
- # ---- multiprocessing
- start = time.time()
- f.seek(0)
- processer = ThreadOrProcess(f, False)
- print(len(dict(processer.lookup)))
- print("mp approach took: %3.3f seconds" % (time.time() - start))
Advertisement
Add Comment
Please, Sign In to add comment