Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def write_to_csv(queue):
- file_path = os.path.join(os.getcwd(), 'test_dir', "writer.csv")
- ofile = open(file_path, "w")
- job_writer = csv.writer(ofile, delimiter='a')
- while 1:
- line = queue.get()
- if line == 'kill':
- print("Kill Signal received")
- break
- if line:job_writer.writerow([str(line).strip()])
- ofile.close()
- def worker_main(file_queue, writer_queue):
- print os.getpid(),"working"
- while not file_queue.empty():
- file_name = file_queue.get(True)
- # somewhere in process_file writer_queue.put(line_resp) is called
- # for every line in file_name
- process_file(file_name, writer_queue)
- if __name__ == "__main__":
- file_queue = multiprocessing.Queue()
- output_queue = multiprocessing.Queue()
- writer_pool = multiprocessing.Pool(1, write_to_csv, (output_queue,))
- cwd = os.getcwd()
- test_dir = 'test_dir'
- file_list = os.listdir(os.path.join(cwd, test_dir))
- for file_name in file_list:
- file_queue.put(file_name)
- reader_pool = multiprocessing.Pool(3, worker_main, (file_queue, output_queue))
- reader_pool.close()
- reader_pool.join()
- output_queue.put("kill")
- print("Finished execution")
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement