Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- class data_reader():
- def __init__(self,paths_list,queue_limit = 1,parallel = True):
- print 'data reader constructor'
- self.paths = paths_list
- self.count = len(paths_list)
- self.async = async
- self.dict = {};
- self.lock = multiprocessing.Lock()
- self.queue = multiprocessing.Queue(queue_limit)
- if parallel:
- self.child = multiprocessing.Process(target = self.load_paths,args = (self.paths,self.queue,))
- self.child.daemon = True
- self.child.start()
- print 'child started'
- else:
- self.load_paths(self.paths,self.queue)
- def get_next():
- return self.queue.get()
- data_train = data_reader(train_paths_list)
- data_valid = data_reader(valid_paths_list)
- data_test = data_reader(test_paths_list)
- 'data reader constructor'
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement