Advertisement
Guest User

Untitled

a guest
Apr 7th, 2020
320
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 7.04 KB | None | 0 0
  1. from sqlalchemy import create_engine
  2. import json
  3. import requests
  4. import logging
  5. from logging.handlers import RotatingFileHandler
  6. import sys
  7.  
  8. connector_id = "mysql_v1_db_sif_canvas"
  9.  
  10. logger = logging.getLogger(connector_id)
  11. hdlr = RotatingFileHandler('./logs/' + connector_id + '.log', maxBytes=10485760, backupCount=10)
  12. formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
  13. hdlr.setFormatter(formatter)
  14. logger.addHandler(hdlr)
  15.  
  16.  
  17. def error_handling():
  18.     return 'Error: {}. {}, line: {}'.format(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2].tb_lineno)
  19.  
  20.  
  21. def execute_mysql(db_conn, query):
  22.     # db_conn.connect()
  23.     query = db_conn.execute(query)
  24.     # db_conn.close()
  25.     return [dict(zip(tuple(query.keys()), i)) for i in query.cursor]
  26.  
  27.  
  28. def execute(manifest):
  29.     try:
  30.         log_level = manifest["log_level"]
  31.         if log_level == "info" or log_level == "Info" or log_level == "INFO":
  32.             logger.setLevel(logging.INFO)
  33.         elif log_level == "warning" or log_level == "Warning" or log_level == "WARNING" or log_level == "WARN" \
  34.                 or log_level == "warn" or log_level == "Warn":
  35.             logger.setLevel(logging.WARNING)
  36.         else:
  37.             logger.setLevel(logging.ERROR)
  38.  
  39.         db_engine = create_engine('mysql+pymysql://' + manifest["connector_config"]["db_username"]["value"] + ':'
  40.                                   + manifest["connector_config"]["db_password"]["value"] + '@'
  41.                                   + manifest["connector_config"]["db_address"]["value"] + '/'
  42.                                   + manifest["connector_config"]["db_name"]["value"], pool_recycle=3600)
  43.         db_conn = db_engine.connect()
  44.  
  45.         response = None
  46.  
  47.         if manifest["job_type"] == 'read':
  48.             fieldmapping = manifest["job_parameters"]["field_mapping"]
  49.             if "unique_id" in manifest["job_parameters"]:
  50.                 unique_id = manifest["job_parameters"]["unique_id"]
  51.             else:
  52.                 unique_id = ""
  53.  
  54.             sqlfields = []
  55.             for k in fieldmapping.keys():
  56.                 sqlfields.append(" " + k + " AS " + fieldmapping[k] + " ")
  57.  
  58.             fields = ",".join(sqlfields)
  59.  
  60.             print(fields)
  61.  
  62.             filter_list = []
  63.             filter_string = ""
  64.  
  65.             if "filters" in manifest["job_parameters"].keys():
  66.                 filters = manifest["job_parameters"]["filters"]
  67.                 andfilter: object
  68.                 for andfilter in filters:
  69.                     filter_list_or = []
  70.                     for orfilter in andfilter:
  71.                         filter_list_or.append(
  72.                             orfilter["field"] + " " + orfilter["operator"] + " " +
  73.                             orfilter["value"].replace("---quote---", "'") + " ")
  74.                     if len(filter_list_or) > 0:
  75.                         filter_list.append("(" + " or ".join(filter_list_or) + ")")
  76.                 if len(filter_list) > 0:
  77.                     filter_string = " where " + " and ".join(filter_list)
  78.  
  79.             if manifest["recordtype"] == 'Users':
  80.                 query = "select " + fields + " from " + manifest["job_parameters"]["table_name"] + filter_string
  81.             elif manifest["recordtype"] == 'Courses':
  82.                 query = "select " + fields + " from " + manifest["job_parameters"]["table_name"] + filter_string
  83.             elif manifest["recordtype"] == 'Sections':
  84.                 query = "select " + fields + " from " + manifest["job_parameters"]["table_name"] + filter_string
  85.             elif manifest["recordtype"] == 'Enrolments':
  86.                 query = "select " + fields + " from " + manifest["job_parameters"]["table_name"] + filter_string
  87.             else:
  88.                 return {"error": {"connector": "invalid recordtype"}}
  89.  
  90.             query_result = execute_mysql(db_conn, query)
  91.             records = []
  92.  
  93.             for item in query_result:
  94.                 if unique_id == "":
  95.                     record = {"data": item}
  96.                     # print(record)
  97.                     records.append(record)
  98.                 else:
  99.                     record = {"data": item, "id": item[unique_id]}
  100.                     # print(record)
  101.                     records.append(record)
  102.  
  103.             manifest["records"] = records
  104.             logger.info("connector post manifest: " + str(json.dumps(manifest)))
  105.  
  106.             response = requests.request("POST", manifest["core_config"]["core_api_url"],
  107.                              headers={
  108.                                 "Content-Type": "application/json",
  109.                                 "X-Secret": manifest["token"],
  110.                                 "job_id": str(manifest["job_id"]),
  111.                                 "tenant_id": manifest["tenant_id"]
  112.                              },
  113.                              data=json.dumps(manifest)
  114.                              )
  115.  
  116.         else:
  117.             return {"error": {"connector": "invalid job_type"}}
  118.  
  119.         if response.status_code < 300:
  120.             return {"success": {"core_api_status_code": str(response.status_code)}}
  121.         else:
  122.             return {"error": {"core_api_status_code": str(response.status_code)}}
  123.  
  124.  
  125.     except:
  126.         logger.error("exception: " + str(error_handling()))
  127.         return {"error": {"exception: ": str(error_handling())}}
  128.  
  129.  
  130. manifest = {
  131.     "tenant_id": "reciprocity_ppd",
  132.     "log_level": "info",
  133.     "recordtype": "Users",
  134.     "job_type": "read",
  135.     "core_config": {
  136.         "core_api_url": "https://reciprocity.saasyan.com.au/api/v1.0/connector/records"
  137.     },
  138.     "token": "3c6faf72-7f96-4870-a035-35e1435df7f3",
  139.     "job_id": 2729,
  140.     "records": [],
  141.     "end_of_list": 1,
  142.     "job_status": "success",
  143.     "connector_config": {
  144.         "db_name": {
  145.             "type": "string",
  146.             "value": "assure",
  147.             "required": "true",
  148.             "description": "Database Name"
  149.         },
  150.         "db_port": {
  151.             "type": "integer",
  152.             "value": "3306",
  153.             "required": "true",
  154.             "description": "Database Port"
  155.         },
  156.         "db_address": {
  157.             "type": "string",
  158.             "value": "10.12.128.100",
  159.             "required": "true",
  160.             "description": "Database Address"
  161.         },
  162.         "db_password": {
  163.             "type": "password",
  164.             "value": "foQBGUPzTbFiFugJa2sC6abdGuiszbvh",
  165.             "required": "true",
  166.             "description": "Database Password"
  167.         },
  168.         "db_username": {
  169.             "type": "string",
  170.             "value": "root",
  171.             "required": "true",
  172.             "description": "Database Username"
  173.         }
  174.     },
  175.     "job_parameters": {
  176.         "unique_id": "user_id",
  177.         "table_name": "sis_users",
  178.         "field_mapping": {
  179.             "lower(email)": "email",
  180.             "user_id": "user_id",
  181.             "lower(login_id)": "login_id",
  182.             "last_name": "last_name",
  183.             "first_name": "first_name"
  184.         }
  185.     },
  186.     "connector_id": "mysql_v1_db_sif_canvas"
  187. }
  188.  
  189. print(execute(manifest))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement