Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # Splits file every (n) lines and return an array of part files paths
- def split(file_path, n):
- part_file_list = []
- with open(file_path) as text_file:
- i = 0
- part = 0
- for line in text_file:
- part_file_path = text_file_path + '.part_' + str(part)
- open(part_file_path, 'a').write(line)
- if part_file_path not in part_file_list:
- part_file_list.append(part_file_path)
- i += 1
- if(i > n):
- i = 0
- part += 1
- return part_file_list
- # Insert into database
- def insert_into_db(conn, file_path, table, fields, static_col, static_col_val):
- conn.query('LOAD DATA LOCAL INFILE \'' + file_path + '\' INTO TABLE ' + table + ' FIELDS TERMINATED BY \\t LINES TERMINATED BY \\n (' + fields + ') SET ' + static_col + '=' + static_col_val + ';')
- def main()
- # MySQL details
- mysql_host = str(raw_input('MySQL host: '))
- mysql_user = str(raw_input('MySQL user: '))
- mysql_pass = str(raw_input('MySQL password: '))
- mysql_db = str(raw_input('MySQL database: '))
- # Options
- file_path = str(raw_input('File path: '))
- table = str(raw_input('Table name: '))
- fields = str(raw_input('Fields (without parenthesis): '))
- static_col = str(raw_input('Static column name: '))
- static_col_val = str(raw_input('Static column value: '))
- chunk_size = int(raw_input('Chunk size: '))
- print 'Trying to connect MySQL database'
- try:
- # Connect
- conn = mysql.connect(mysql_host, mysql_user, mysql_pass, mysql_db)
- # Split files
- chunked_files = split(file_path, chunk_size)
- # Insert them
- for chunk in chunked_files:
- insert_into_db(conn, chunk, table, fields, static_col, static_col_val)
- except mysql.Error e:
- print e
- main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement