SHARE
TWEET

Untitled

a guest Jun 17th, 2019 110 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. from pathos import multiprocessing as multiprocessing
  2. from functools import partial
  3. from ThrdPool import *
  4.  
  5. class APICall():
  6.      def __init__(self):
  7.      self.host = config.get('splunk_search_section', 'host_name')
  8.      self.port = config.get('splunk_search_section', 'host_port')
  9.      self.username = config.get('splunk_search_section', 'user_name')
  10.      self.password = config.get('splunk_search_section', 'password')
  11.  def process(self):
  12.      company_list = self.fetch_onboarded_companies_from_customer_csv()
  13.      saved_search_list_file = os.path.join(code_dir_path, "resources\saved_search_template.txt")
  14.      try:
  15.          with open(saved_search_list_file, "r") as ss_file_pointer:
  16.              saved_search_list = ss_file_pointer.readlines()
  17.      except IOError as ie:
  18.          self.logging.error(f"Error occurred while accessing the Saved Search file reason being , {ie}")
  19.          raise IOError("IO Error occurred while accessing the Saved Search file.")
  20.      finally:
  21.          ss_file_pointer.close()
  22.      # Creating a process pool for each company key, to increase the throughput.
  23.      # Assuming 4 processors to be max optimistically.
  24.      p = multiprocessing.Pool(processes=4)
  25.      # for each_cpy in company_list:
  26.      list_length_company = len(company_list)
  27.      array_of_numbers = [x for x in range(0, list_length_company)]
  28.      ssl = saved_search_list
  29.      cl = company_list
  30.      func = partial(self.processing_saved_search_per_company, ssl, cl)
  31.      p.map(func, array_of_numbers)
  32.      p.close()
  33.      p.join()
  34.  def processing_saved_search_per_company(self, saved_search_list, company_list, each_cpy_index):
  35.      company_key = company_list[each_cpy_index]
  36.      print("Company Key : " + company_key)
  37.      self.logging.info(f"processing the saved search for company {company_key}")
  38.      each_cpy = company_list[each_cpy_index]
  39.      array_of_numbers = [x for x in range(0, len(saved_search_list))]
  40.      # Creating a Thread pool of 5 threads to optimistically increase the throughput of saved search processing.
  41.      thread_pool = ThrdPool(5)
  42.      function1 = partial(self.run_saved_search_per_company, saved_search_list, each_cpy)
  43.      thread_pool.map(function1, array_of_numbers)
  44.      thread_pool.wait_completion()
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top