daily pastebin goal
11%
SHARE
TWEET

Untitled

a guest Sep 14th, 2018 45 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. log.write("Value Error Found for: " + str(my_dict["REQUEST_ID"]) + str(my_dict["RESULT_ID"]))
  2.    
  3. import json
  4. import csv
  5. import os
  6. import codecs
  7. import sys
  8. import datetime
  9. import logging
  10. import traceback
  11. reload(sys)
  12. sys.setdefaultencoding('utf8')
  13.  
  14. def my_get(mdict, key, sub_key):
  15.     if isinstance(mdict, dict):
  16.         mvalue = mdict.get(key)
  17.         if isinstance(mvalue, dict):
  18.             return mvalue.get(sub_key)
  19.         else:
  20.             return mvalue
  21.     else:
  22.         return None
  23.  
  24. def setup_log_file(filename):
  25.     logging.basicConfig(filename = filename,
  26.                         filemode='w',
  27.                         level=logging.DEBUG,
  28.                         format= '%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
  29.                         datefmt='%d-%m-%Y:%H:%M:%S',
  30.                         )
  31.  
  32.  
  33. def log_exception(exp):
  34.     logger = logging.getLogger(__name__)
  35.     logger.error("Failed to read JSON file", exc_info=True)
  36.  
  37.  
  38. file_dir = os.path.normpath('/home/pp/jobs/staging/')
  39. exp_dir = os.path.normpath('/home/pp/jobs/CSV/')
  40. log_dir = os.path.normpath('/home/pp/jobs/Logs/')
  41. exp_file_name = 'emails.csv'
  42. log_file_name = 'log_emails.log'
  43. exp_path = os.path.join(exp_dir, exp_file_name)
  44. log_path = os.path.join(log_dir, log_file_name)
  45. setup_log_file(log_path)
  46.  
  47. my_dict_list =[]
  48. try:
  49.     for f in os.listdir(file_dir):
  50.         if f.endswith('.json') and f.startswith('emails_'):
  51.             file_path = os.path.join(file_dir, f)
  52.             data = open(file_path, 'r')
  53.             for line in data:
  54.                 parsed_data = json.loads(line)
  55.                 for result in parsed_data["result"]:
  56.                     my_dict = {}
  57.                     my_dict["REQUEST_ID"] = parsed_data["requestId"]
  58.                     my_dict["SUCCESS"] = parsed_data["success"]
  59.                     my_dict["RESULT_ID"] = my_get(result,"id",'')
  60.                     my_dict["NAME"] = my_get(result,"name",'')
  61.                     my_dict["DESCRIPTION"] = my_get(result,"description",'')
  62.                     my_dict["STATUS"] = my_get(result,"status",'')
  63.                     my_dict["FOLDER_TYPE"] = my_get(result,"folder","type")
  64.                     my_dict["FOLDER_ID"] = my_get(result,"folder","value")
  65.                     my_dict["FOLDER_NAME"] = my_get(result,"folder","folderName")
  66.                     my_dict["FROM_EMAIL_TYPE"] = my_get(result,"fromEmail","type")
  67.                     my_dict["FROM_EMAIL_VALUE"] = my_get(result,"fromEmail","value")
  68.                     my_dict["FROM_NAME_TYPE"] = my_get(result,"fromName","type")
  69.                     my_dict["FROM_NAME_VALUE"] = my_get(result,"fromName","value")
  70.                     my_dict["REPLY_EMAIL_TYPE"] = my_get(result,"replyEmail","type")
  71.                     my_dict["REPLY_EMAIL_VALUE"] = my_get(result,"replyEmail","value")
  72.                     my_dict["SUBJECT_TYPE"] = my_get(result,"subject","type")
  73.                     my_dict["SUBJECT_VALUE"] = my_get(result,"subject","value")
  74.                     my_dict["OPERATIONAL"] = my_get(result,"operational",'')
  75.                     my_dict["PUBLISH_TO_MSI"] = my_get(result,"publishToMSI",'')
  76.                     my_dict["TEMPLATE"] = my_get(result,"template",'')
  77.                     my_dict["TEXT_ONLY"] = my_get(result,"textOnly",'')
  78.                     my_dict["URL"] = my_get(result,"url",'')
  79.                     my_dict["WEBVIEW"] = my_get(result,"webView",'')
  80.                     my_dict["CREATED_AT"] = my_get(result,"createdAt",'')
  81.                     my_dict["UPDATED_AT"] = my_get(result,"updatedAt",'')
  82.                     my_dict["WORKSPACE"] = my_get(result,"workspace",'')
  83.                     my_dict_list.append(my_dict)
  84.  
  85.     csv_columns = ["REQUEST_ID","SUCCESS","RESULT_ID","NAME","DESCRIPTION","STATUS","FOLDER_TYPE","FOLDER_ID","FOLDER_NAME","FROM_EMAIL_TYPE","FROM_EMAIL_VALUE","FROM_NAME_TYPE","FROM_NAME_VALUE","REPLY_EMAIL_TYPE","REPLY_EMAIL_VALUE","SUBJECT_TYPE","SUBJECT_VALUE","OPERATIONAL","PUBLISH_TO_MSI","TEMPLATE","TEXT_ONLY","URL","WEBVIEW","CREATED_AT","UPDATED_AT","WORKSPACE"]
  86.     with open(exp_path,'wb') as csvfile:
  87.                    xz = csv.DictWriter(csvfile,fieldnames=csv_columns)
  88.                    headers = {}
  89.                    for n in xz.fieldnames:
  90.                        headers[n] = n
  91.                    xz.writerow(headers)
  92.                    for data in my_dict_list:
  93.                        xz.writerow(data)
  94. except Exception as exp:
  95.     log_exception(exp)
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top