Advertisement
MeowalsoMeow

convoy shipping stage 3

Sep 19th, 2022
27
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.49 KB | None | 0 0
  1. import pandas as pd
  2. import csv
  3. import re
  4. import sqlite3
  5.  
  6.  
  7. # when its runned the second time there will be two [CHECKED], why is this?
  8.  
  9.  
  10. def xlsx_to_csv():
  11. input_file_name = input('Input file name\n')
  12. if input_file_name.endswith('.csv'):
  13. csv_file_name = input_file_name
  14. else:
  15. my_df = pd.read_excel(input_file_name, sheet_name='Vehicles', dtype=str)
  16. # csv_file_name = input_file.removesuffix('.xlsx') + '.csv'
  17. csv_file_name = input_file_name.replace('.xlsx', '.csv')
  18. # print(csv_file_name)
  19. my_df.to_csv(csv_file_name, index=False)
  20. line_num = my_df.shape[0]
  21. print(f'{line_num} line{"s were" if line_num > 1 else " was"} imported to {csv_file_name}')
  22. return csv_file_name
  23.  
  24.  
  25. def clean_csv(csv_file_name):
  26. #csv_checked_name = csv_file_name.removesuffix('.csv') + '[CHECKED]' + '.csv'
  27. csv_checked_name = csv_file_name.removesuffix(
  28. '.csv') + '[CHECKED]' + '.csv' if '[CHECKED]' not in csv_file_name else csv_file_name
  29. # print('csv_checked_name', csv_checked_name)
  30. with open(csv_file_name, 'r') as csv_file, open(csv_checked_name, 'w', encoding='utf-8') as checked_file:
  31. file_reader = csv.reader(csv_file, delimiter=",")
  32. file_writer = csv.writer(checked_file, delimiter=",", lineterminator="\n")
  33. count = 0
  34. wrong_data_count = 0
  35. for line in file_reader:
  36. if count == 0:
  37. file_writer.writerow(line)
  38. else:
  39. for cell in line:
  40. if not cell.isnumeric():
  41. wrong_data_count += 1
  42. line[line.index(cell)] = int(re.findall(r'\d+', cell)[0]) # need to replace the cell in line
  43. file_writer.writerow(line)
  44. count += 1
  45.  
  46. print(f'{wrong_data_count} cells were corrected in {csv_checked_name}')
  47. # print(csv_checked_name, type(csv_checked_name)) returns string
  48. # print(checked_file, type(checked_file)) returns io wrapper
  49. return csv_checked_name
  50.  
  51.  
  52. def csv_to_s3db(cleaned_csv):
  53. database = cleaned_csv.removesuffix('[CHECKED].csv') + '.s3db'
  54. # print('database', database)
  55. conn = sqlite3.connect(database)
  56. cursor_name = conn.cursor()
  57. with open(cleaned_csv, 'r') as csv_file:
  58. file_reader = csv.reader(csv_file, delimiter=",")
  59. count = 0
  60. for line in file_reader:
  61. if count == 0:
  62. cursor_name.execute('''
  63. CREATE TABLE IF NOT EXISTS convoy (
  64. [vehicle_id] INT PRIMARY KEY,
  65. [engine_capacity] INT NOT NULL,
  66. [fuel_consumption] INT NOT NULL,
  67. [maximum_load] INT NOT NULL
  68. );
  69. ''')
  70. else:
  71. cursor_name.execute('''
  72. INSERT INTO convoy (vehicle_id, engine_capacity, fuel_consumption, maximum_load)
  73. VALUES (?,?,?,?);
  74. ''', line)
  75. count += 1
  76. # cursor_name.execute("SELECT * FROM convoy")
  77. # print('table', cursor_name.fetchall())
  78. count -= 1 if count > 0 else 0
  79. conn.commit()
  80. conn.close()
  81. print(f'{count} record{"s were" if count > 1 else " was"} inserted into {database}')
  82.  
  83.  
  84. def main():
  85. file_name = xlsx_to_csv()
  86. # print('filename', file_name)
  87. cleaned_csv = clean_csv(file_name)
  88. # print('cleanned csv', cleaned_csv)
  89. csv_to_s3db(cleaned_csv)
  90.  
  91.  
  92. if __name__ == '__main__':
  93. main()
  94.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement