Advertisement
Guest User

Untitled

a guest
May 1st, 2018
117
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 4.52 KB | None | 0 0
  1. Invalid Argument error on the python code
  2.  
  3. I am beginner to python, and I have this code to decompress log file, but I have the error Invalid argument. I don't know why I got this error, in my opinion, I think it is because the log file is too big, cause I am scanning a log file which is 2gb file. But I have no idea how to fix the error. Please help, thank you. And below is my code with the error:
  4.  
  5. import glob
  6. import gzip
  7. import os
  8. import pymysql
  9. import logging
  10. # path to gz directory
  11. GZ_DIR = '/Users/kiya/Desktop/mysql/csv'
  12. # Database Infomation
  13. DB_HOST='locahost'
  14. DB_USER='dbuser'
  15. DB_PASS='dbPassword'
  16. DB_NAME='dbname'
  17. LOGFILE="exception.log"
  18. def csv_reader(file, header=False):
  19.    import csv
  20.    with open(file, "r") as f:
  21.        reader = csv.reader(f)
  22.        if header:
  23.            next(reader)
  24.        for row in reader:
  25.            yield row
  26. def import_sql(filename, dbHostName, dbUser, dbPassword, databaseName):
  27.    db = pymysql.connect(host=dbHostName,
  28.                         user=dbUser,
  29.                         password=dbPassword,
  30.                         db=databaseName,
  31.                         charset='utf8')
  32.  
  33.    for row in csv_reader(filename, False):
  34.        # prepare a cursor object using cursor() method
  35.        with db.cursor() as cursor:
  36.            if row[3] == "THREAT" and row[4] == "url":
  37.                sql = ("INSERT INTO PADIAGDB.url ("
  38.                       "Domain,Receive_Time,Serial,Type,Threat_Content_Type,"
  39.                       "Config_Version,Generate_Time,Source_address,Destination_address,"
  40.                       "NAT_Source_IP,NAT_Destination_IP,Rule,Source_User,"
  41.                       "Destination_User,Application,Virtual_System,Source_Zone,"
  42.                       "Destination_Zone,Inbound_Interface,Outbound_Interface,Log_Action,"
  43.                       "Time_Logged,Session_ID,Repeat_Count,Source_Port,Destination_Port,"
  44.                       "NAT_Source_Port,NAT_Destination_Port,Flags,IP_Protocol,Action,"
  45.                       "URL_Filename,Threat_Content_Name,Category,Severity,Direction,"
  46.                       "Sequence_Number,Action_Flags,Source_Country,Destination_Country,"
  47.                       "cpadding,contenttype,pcap_id,filedigest,cloud,url_idx,user_agent,"
  48.                       "filetype,xff,referer,sender,subject,recipient,reportid,"
  49.                       "dg_hier_level_1,dg_hier_level_2,dg_hier_level_3,dg_hier_level_4,"
  50.                       "Virtual_System_Name,Device_Name,file_url )"
  51.                       ""
  52.                       "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"
  53.                       "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"
  54.                       "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s"
  55.                       "); ")
  56.            elif row[3] == "SYSTEM":
  57.                sql = ("INSERT INTO PADIAGDB.system ("
  58.                       "Domain,Receive_Time,Serial,Type,Threat_Content_Type,Config_Version,"
  59.                       "Generate_Time,Virtual_System,Event_ID,Object,fmt,id,module,Severity,"
  60.                       "Description,Sequence_Number,Action_Flags,dg_hier_level_1,"
  61.                       "dg_hier_level_2,dg_hier_level_3,dg_hier_level_4,Virtual_System_Name,"
  62.                       "Device_Name )"
  63.                       ""
  64.                       "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,"
  65.                       "%s,%s,%s );")
  66.            else:
  67.                continue
  68.            try:
  69.                cursor.execute('SET foreign_key_checks = 0')
  70.                # Execute the SQL command
  71.                r = cursor.execute(sql, row)
  72.  
  73.                #Commit your changes in the database
  74.                cursor.execute('SET foreign_key_checks = 1')
  75.                db.commit()
  76.            except Exception as e:
  77.                logging.exception(e)
  78.                db.rollback()
  79.     # disconnect from server
  80.    db.close()
  81. gz_files = (gz for gz in glob.glob(os.path.join(GZ_DIR, '*.gz')))
  82. for gz_file in gz_files:
  83.    with gzip.open(gz_file, 'rb') as in_file:
  84.        s = in_file.read()
  85.    sql_file = gz_file[:-3]
  86.    sql_file = sql_file[:-4] + '.csv'
  87.    with open(sql_file, 'wb') as out_file:
  88.        out_file.write(s)
  89.    import_sql(out_file, DB_HOST, DB_USER, DB_PASS, DB_NAME)
  90.    os.remove(sql_file)
  91.  
  92. This is the error I got:
  93.  
  94. Traceback (most recent call last):
  95.  File "/Users/kiya/Desktop/mysql/csv/sql3.py", line 180, in <module>
  96.    out_file.write(s)
  97. OSError: [Errno 22] Invalid argument
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement