Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from pathos import multiprocessing as multiprocessing
- from functools import partial
- from ThrdPool import *
- class APICall():
- def __init__(self):
- self.host = config.get('splunk_search_section', 'host_name')
- self.port = config.get('splunk_search_section', 'host_port')
- self.username = config.get('splunk_search_section', 'user_name')
- self.password = config.get('splunk_search_section', 'password')
- def process(self):
- company_list = self.fetch_onboarded_companies_from_customer_csv()
- saved_search_list_file = os.path.join(code_dir_path, "resources\saved_search_template.txt")
- try:
- with open(saved_search_list_file, "r") as ss_file_pointer:
- saved_search_list = ss_file_pointer.readlines()
- except IOError as ie:
- self.logging.error(f"Error occurred while accessing the Saved Search file reason being , {ie}")
- raise IOError("IO Error occurred while accessing the Saved Search file.")
- finally:
- ss_file_pointer.close()
- # Creating a process pool for each company key, to increase the throughput.
- # Assuming 4 processors to be max optimistically.
- p = multiprocessing.Pool(processes=4)
- # for each_cpy in company_list:
- list_length_company = len(company_list)
- array_of_numbers = [x for x in range(0, list_length_company)]
- ssl = saved_search_list
- cl = company_list
- func = partial(self.processing_saved_search_per_company, ssl, cl)
- p.map(func, array_of_numbers)
- p.close()
- p.join()
- def processing_saved_search_per_company(self, saved_search_list, company_list, each_cpy_index):
- company_key = company_list[each_cpy_index]
- print("Company Key : " + company_key)
- self.logging.info(f"processing the saved search for company {company_key}")
- each_cpy = company_list[each_cpy_index]
- array_of_numbers = [x for x in range(0, len(saved_search_list))]
- # Creating a Thread pool of 5 threads to optimistically increase the throughput of saved search processing.
- thread_pool = ThrdPool(5)
- function1 = partial(self.run_saved_search_per_company, saved_search_list, each_cpy)
- thread_pool.map(function1, array_of_numbers)
- thread_pool.wait_completion()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement