dreadngel

multiprocessing sample

Aug 16th, 2019
177
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.52 KB | None | 0 0
  1. import multiprocessing
  2. import xml.etree.ElementTree as ET
  3. import time
  4. import sys
  5. import re
  6.  
  7. def work_processor(host_item, output_queue):
  8.     # print("Processing ->" + host_item.attrib['name'])
  9.     block_summary = {}
  10.     block_issues = []    
  11.     try:
  12.         # !!! big work here with host_item!!!
  13.         result = {
  14.                     "summary": block_summary,
  15.                     "issues": block_issues
  16.                 }
  17.         return result
  18.     except Exception as e:
  19.         result = {
  20.                     "summary": block_summary,
  21.                     "issues": block_issues,
  22.                     "error" : str(e)
  23.             }
  24.     output_queue.put(result)
  25.  
  26.  
  27. def start_point(xml_report_file):
  28.     try:
  29.         dom = ET.parse(open(xml_report_file, "r"))
  30.         root = dom.getroot()
  31.     except Exception as e:
  32.         print("Exception - {}".format(str(e)))
  33.         return None
  34.    
  35.  
  36.     host_report_items = [host_report for root_items in root if root_items.tag == 'Report' for host_report in root_items]
  37.     print("Data loaded...")
  38.  
  39.     start_time = time.time()
  40.     result_data = multiprocessing.Queue(0)    
  41.     process_list = [multiprocessing.Process(target=work_processor, args=(item, result_data,)) for item in host_report_items]
  42.    
  43.     for p in process_list:
  44.         p.start()
  45.  
  46.     print("Processes launched... ")
  47.  
  48.     for p in process_list:
  49.         p.join()
  50.  
  51.     print("Processes finished... ")
  52.  
  53.     results = [result_data.get() for _p in process_list]
  54.     print("Time consumed:{}".format(time.time() - start_time))
  55.     print(len(results))
  56.    
  57. if __name__== '__main__':
  58.     file_name = "data.xml"
  59.     start_point(file_name)
Advertisement
Add Comment
Please, Sign In to add comment