Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import csv
- import glob
- import gzip
- import logging
- import os
- import pymysql
- import subprocess
- # path to gz directory
- GZ_DIR = "/Users/kiya/Desktop/mysql/csv"
- # Database Infomation
- DB_HOST = 'host'
- DB_USER = 'dbuser'
- DB_PASS = 'dppass'
- DB_NAME = 'dbname'
- LOGFILE = "exception.log"
- def csv_reader(file, header=False):
- with open(file, "r") as f:
- reader = csv.reader(f)
- if header:
- next(reader)
- for row in reader:
- yield row
- def import_sql(filename, dbHostName, dbUser, dbPassword, databaseName):
- db = pymysql.connect(host=dbHostName,
- user=dbUser,
- password=dbPassword,
- db=databaseName,
- charset='utf8')
- for row in csv_reader(filename, False):
- # prepare a cursor object using cursor() method
- with db.cursor() as cursor:
- if row[3] == "THREAT" and row[4] == "url":
- sql = #query
- else:
- continue
- try:
- cursor.execute('SET foreign_key_checks = 0')
- # Execute the SQL command
- r = cursor.execute(sql, row)
- # Commit your changes in the database
- cursor.execute('SET foreign_key_checks = 1')
- db.commit()
- except Exception as e:
- logging.exception(e)
- db.rollback()
- # disconnect from server
- db.close()
- gz_files = (gz for gz in glob.glob(os.path.join(GZ_DIR, '*.gz')))
- for gz_file in gz_files:
- sql_file = gz_file[:-3]
- sql_file = sql_file[:-4] + '.csv'
- with open(sql_file, 'wb') as out_file:
- with gzip.open(gz_file, 'rb') as in_file:
- while True:
- chunk = in_file.read(1024)
- if not chunk:
- break
- out_file.write(chunk)
- import_sql(out_file, DB_HOST, DB_USER, DB_PASS, DB_NAME)
- # os.remove(sql_file)
Add Comment
Please, Sign In to add comment