Guest User

Untitled

a guest
May 10th, 2018
111
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.17 KB | None | 0 0
  1. import csv
  2. import glob
  3. import gzip
  4. import logging
  5. import os
  6. import pymysql
  7. import subprocess
  8. # path to gz directory
  9. GZ_DIR = "/Users/kiya/Desktop/mysql/csv"
  10. # Database Infomation
  11. DB_HOST = 'host'
  12. DB_USER = 'dbuser'
  13. DB_PASS = 'dppass'
  14. DB_NAME = 'dbname'
  15. LOGFILE = "exception.log"
  16. def csv_reader(file, header=False):
  17. with open(file, "r") as f:
  18. reader = csv.reader(f)
  19. if header:
  20. next(reader)
  21. for row in reader:
  22. yield row
  23. def import_sql(filename, dbHostName, dbUser, dbPassword, databaseName):
  24. db = pymysql.connect(host=dbHostName,
  25. user=dbUser,
  26. password=dbPassword,
  27. db=databaseName,
  28. charset='utf8')
  29.  
  30. for row in csv_reader(filename, False):
  31. # prepare a cursor object using cursor() method
  32. with db.cursor() as cursor:
  33. if row[3] == "THREAT" and row[4] == "url":
  34. sql = #query
  35. else:
  36. continue
  37.  
  38. try:
  39. cursor.execute('SET foreign_key_checks = 0')
  40. # Execute the SQL command
  41. r = cursor.execute(sql, row)
  42.  
  43. # Commit your changes in the database
  44. cursor.execute('SET foreign_key_checks = 1')
  45. db.commit()
  46.  
  47.  
  48. except Exception as e:
  49. logging.exception(e)
  50. db.rollback()
  51.  
  52. # disconnect from server
  53. db.close()
  54. gz_files = (gz for gz in glob.glob(os.path.join(GZ_DIR, '*.gz')))
  55. for gz_file in gz_files:
  56. sql_file = gz_file[:-3]
  57. sql_file = sql_file[:-4] + '.csv'
  58. with open(sql_file, 'wb') as out_file:
  59. with gzip.open(gz_file, 'rb') as in_file:
  60. while True:
  61. chunk = in_file.read(1024)
  62. if not chunk:
  63. break
  64. out_file.write(chunk)
  65. import_sql(out_file, DB_HOST, DB_USER, DB_PASS, DB_NAME)
  66. # os.remove(sql_file)
Add Comment
Please, Sign In to add comment