Advertisement
Guest User

Untitled

a guest
May 15th, 2016
65
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.88 KB | None | 0 0
  1. # Splits file every (n) lines and return an array of part files paths
  2. def split(file_path, n):
  3.  
  4.     part_file_list = []
  5.  
  6.     with open(file_path) as text_file:
  7.         i = 0
  8.         part = 0
  9.         for line in text_file:
  10.             part_file_path = text_file_path + '.part_' + str(part)
  11.             open(part_file_path, 'a').write(line)
  12.  
  13.             if part_file_path not in part_file_list:
  14.                 part_file_list.append(part_file_path)
  15.            
  16.             i += 1
  17.             if(i > n):
  18.                 i = 0
  19.                 part += 1
  20.  
  21.     return part_file_list
  22.  
  23. # Insert into database
  24. def insert_into_db(conn, file_path, table, fields, static_col, static_col_val):
  25.     conn.query('LOAD DATA LOCAL INFILE \'' + file_path + '\' INTO TABLE ' + table + ' FIELDS TERMINATED BY \\t LINES TERMINATED BY \\n (' + fields + ') SET ' + static_col + '=' + static_col_val + ';')
  26.  
  27. def main()
  28.  
  29.     # MySQL details
  30.     mysql_host = str(raw_input('MySQL host: '))
  31.     mysql_user = str(raw_input('MySQL user: '))
  32.     mysql_pass = str(raw_input('MySQL password: '))
  33.     mysql_db = str(raw_input('MySQL database: '))
  34.  
  35.     # Options
  36.     file_path = str(raw_input('File path: '))
  37.     table = str(raw_input('Table name: '))
  38.     fields = str(raw_input('Fields (without parenthesis): '))
  39.     static_col = str(raw_input('Static column name: '))
  40.     static_col_val = str(raw_input('Static column value: '))
  41.     chunk_size = int(raw_input('Chunk size: '))
  42.  
  43.     print 'Trying to connect MySQL database'
  44.     try:
  45.         # Connect
  46.         conn = mysql.connect(mysql_host, mysql_user, mysql_pass, mysql_db)
  47.        
  48.         # Split files
  49.         chunked_files = split(file_path, chunk_size)
  50.  
  51.         # Insert them
  52.         for chunk in chunked_files:
  53.             insert_into_db(conn, chunk, table, fields, static_col, static_col_val)
  54.  
  55.     except mysql.Error e:
  56.         print e
  57.  
  58. main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement