Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import threading
- import queue
- from tkinter import *
- from tkinter import filedialog
- import tkinter.ttk as ttk
- import os
- import json
- import regex as ra
- import urllib.request
- import time
- class GuiPart:
- def __init__(self, master, queue, endCommand, parseCommand, openCommand):
- self.queue = queue
- # Set up the GUI
- Label(root, text="JSON PARSER :) ").grid(row=0, column=0, columnspan=8, sticky="E")
- Label(root, text="LOG Files.TXT").grid(row=1, column=0) # Input section row 0
- Label(root, text="input").grid(row=1, column=1, sticky="E")
- self.e0 = Entry(root, state='disabled')
- self.e0.grid(row=1, column=2, sticky="WE")
- self.b0 = Button(root, text="...", command=lambda: openCommand(0, 1)).grid(row=1, column=3, sticky="E")
- Label(root, text='start,stop index').grid(row=0,column=0)
- self.e1 = Entry(root, width=14)
- self.e1.grid(row=0, column=1, columnspan=2, sticky="W")
- self.e2 = Entry(root, width=15)
- self.e2.grid(row=0, column=2, columnspan=2, sticky="E")
- # self.e1 = Entry(root, state='disabled')
- # self.e1.grid(row=1, column=3, sticky="WE")
- # self.e2 = Entry(root, state='disabled')
- # self.e2.grid(row=1, column=4, sticky="WE")
- Label(root, text="JSON log").grid(row=3, column=4, sticky="W") # JSON log section
- Label(root, text="insert").grid(row=3, column=5, sticky="W") # e4,b4 for insert json e5,b5= save destination
- self.e4 = Entry(root, state='disabled')
- self.e4.grid(row=3, column=6, sticky="WE")
- self.b4 = Button(root, text="...", command=lambda: openCommand(0)).grid(row=3, column=7, sticky="E")
- Label(root, text="Location log").grid(row=3, column=0, sticky="W") # IP Location section e6,b6 for insert file, e7,b7=destination
- self.e6 = Entry(root, state='disabled')
- self.e6.grid(row=3, column=2, sticky="WE")
- self.b6 = Button(root, text="...", command=lambda: openCommand(1)).grid(row=3, column=3, sticky="E")
- Label(root, text="<=======================================================>").grid(row=5, column=0,
- columnspan=8, sticky="E")
- self.bp = Button(root, text="parse", command=parseCommand).grid(row=6, column=0, sticky="WE")
- self.bp = Button(root, text="stop", command=endCommand).grid(row=6, column=1,columnspan=2, sticky="W")
- self.numtask = Label(root, text="")
- self.numtask = Label(root, text="")
- self.numtask.grid(row=6, column=1, stick='W')
- failedtask = Label(root)
- failedtask.grid(row=6, column=2, sticky="E")
- self.mpb = ttk.Progressbar(root, orient="horizontal", mode="determinate")
- self.mpb.grid(row=6, column=3, columnspan=5, sticky="WE")
- Label(root, text="=======================================================").grid(row=7, column=0, columnspan=8,
- sticky="E")
- Label(root, text="STATUS:").grid(row=8, column=0, sticky="E")
- self.statusl = Label(root, text="")
- self.statusl.grid(row=8, column=1, columnspan=7)
- Label(root, text="=======================================================").grid(row=9, column=0, columnspan=8,
- sticky="E")
- class ThreadedClient:
- """
- Launch the main part of the GUI and the worker thread. periodicCall and
- endApplication could reside in the GUI part, but putting them here
- means that you have all the thread controls in a single place.
- """
- def __init__(self, master):
- """
- Start the GUI and the asynchronous threads. We are in the main
- (original) thread of the application, which will later be used by
- the GUI. We spawn a new thread for the worker.
- """
- self.master = master
- # Create the queue
- self.queue = queue.Queue()
- # Set up the GUI part
- self.gui = GuiPart(master, self.queue, self.endApplication, self.parse, self.openfile)
- # Set up the thread to do asynchronous I/O
- # More can be made if necessary
- self.key = '95a6c0b2322235de615bcd69a539e604'
- self.loc_data={}
- self.log_data = []
- self.ip_add = []
- self.jsonwrite = []
- self.get_ip = []
- self.barvar=0
- self.running = 1
- self.dumbvalue=0
- # Start the periodic call in the GUI to check if the queue contains
- # anything
- self.periodicCall()
- def openfile(self, loc, txt=0):
- # try:
- if txt==0:
- root.filename = filedialog.askopenfilename(initialdir="/home/isara/PycharmProject/cantforgether",
- title="Select JSON",
- filetypes=(("json files", "*.json"), ("all files", "*.*")))
- if root.filename == '':
- # status = Label(root, text="No file selected")
- self.gui.statusl['text'] = " You didn't select any json "
- if loc:
- self.gui.e6['state'] = 'normal'
- self.gui.e6.delete(0, END)
- self.gui.e6.insert(0, root.filename)
- self.gui.e6['state'] = 'disabled'
- else:
- self.gui.e4['state'] = 'normal'
- self.gui.e4.delete(0, END)
- self.gui.e4.insert(0, root.filename)
- self.gui.e4['state'] = 'disabled'
- else:
- root.filename = filedialog.askopenfilename(initialdir="/home/isara/PycharmProject/cantforgether",
- title="Select TXT",
- filetypes=(("txt files", "*.txt"), ("all files", "*.*")))
- if root.filename == '':
- self.gui.statusl['text'] = " You didn't select any txt "
- self.gui.e0['state'] = 'normal'
- self.gui.e0.delete(0, END)
- self.gui.e0.insert(0, root.filename)
- self.gui.e0['state'] = 'disabled'
- # except:
- # self.statusl['text'] = "CANNOT OPEN file :("
- def periodicCall(self):
- """
- Check every 100 ms if there is something new in the queue.
- """
- self.gui.mpb["value"]=self.barvar
- text= 'dumped data ' + str(self.barvar) + 'from total of ' + str(self.dumbvalue)
- self.gui.statusl['text']=text
- if not self.running:
- # This is the brutal stop of the system. You may want to do
- # some cleanup before actually shutting it down.
- time.sleep(1)
- import sys
- sys.exit(1)
- # root.destroy()
- self.master.after(100, self.periodicCall)
- def workerThread1(self):
- while self.running:
- if self.log_data ==[]:
- time.sleep(0.2)
- else:
- data=str(self.log_data.pop())[2:-1]
- regex = '(.+[\s]*) (-) (.+[\s]*) \[(.*?)\] \"(.*?)\" (.+) (.+)'
- grouplist = ra.match(regex, data).groups()
- if grouplist[0] in self.ip_add:
- ip = self.loc_data[grouplist[0]][0]
- lat = self.loc_data[grouplist[0]][1]
- long = self.loc_data[grouplist[0]][2]
- country = self.loc_data[grouplist[0]][3]
- if lat == None or long == None:
- self.barvar += 1
- continue
- else:
- jsonobj={"ip":grouplist[0], "hyphen":grouplist[1], "frank":grouplist[2],
- "time":grouplist[3], "reqline":grouplist[4], "HTTPcode":grouplist[5],
- "bytesize":grouplist[6], "latitude":lat, "longitude":long, "country_name":country}
- self.jsonwrite.append(jsonobj)
- else:
- iplist=[grouplist[0],grouplist[1],grouplist[2],grouplist[3],grouplist[4],grouplist[5],grouplist[6]]
- self.get_ip.append(iplist)
- def workerThread2(self):
- while self.running:
- if self.get_ip==[]:
- time.sleep(0.2)
- else:
- try:
- # print('T2')
- data2=self.get_ip.pop()
- url = 'http://api.ipstack.com/' + data2[0] + '?access_key=' + self.key
- contents = urllib.request.urlopen(url).read()
- output = contents.decode('utf-8')
- getjson = json.loads(output)
- ip= getjson['ip']
- country = getjson['country_name']
- lat = getjson['latitude']
- long = getjson['longitude']
- self.loc_data[data2[0]]=[ip,lat,long,country]
- self.ip_add.append(ip)
- if lat == None or long == None:
- self.barvar += 1
- self.loc_data[ip]=[ip,lat,long,country]
- with open(self.gui.e6.get(), 'w+') as outfile:
- json.dump(self.loc_data, outfile, indent=2)
- continue
- else:
- jsonobj = {"ip": ip, "hyphen": data2[1], "frank": data2[2],
- "time": data2[3], "reqline": data2[4], "HTTPcode": data2[5],
- "bytesize": data2[6], "latitude": lat, "longitude": long, "country_name": country}
- self.jsonwrite.append(jsonobj)
- except:self.barvar+=1
- def workerThread3(self):
- while self.running:
- if self.get_ip == []:
- time.sleep(0.2)
- else:
- try:
- # print('T3')
- data2 = self.get_ip.pop()
- url = 'http://api.ipstack.com/' + data2[0] + '?access_key=' + self.key
- contents = urllib.request.urlopen(url).read()
- output = contents.decode('utf-8')
- getjson = json.loads(output)
- ip = getjson['ip']
- country = getjson['country_name']
- lat = getjson['latitude']
- long = getjson['longitude']
- self.loc_data[data2[0]] = [ip, lat, long, country]
- self.ip_add.append(ip)
- if lat == None or long == None:
- self.barvar += 1
- self.loc_data[ip] = [ip, lat, long, country]
- with open(self.gui.e6.get(), 'w+') as outfile:
- json.dump(self.loc_data, outfile, indent=2)
- continue
- else:
- jsonobj = {"ip": ip, "hyphen": data2[1], "frank": data2[2],
- "time": data2[3], "reqline": data2[4], "HTTPcode": data2[5],
- "bytesize": data2[6], "latitude": lat, "longitude": long, "country_name": country}
- self.jsonwrite.append(jsonobj)
- except:
- self.barvar += 1
- def workerThread4(self):
- while self.running:
- if self.get_ip == []:
- time.sleep(0.2)
- else:
- try:
- # print('T4')
- data2 = self.get_ip.pop()
- url = 'http://api.ipstack.com/' + data2[0] + '?access_key=' + self.key
- contents = urllib.request.urlopen(url).read()
- output = contents.decode('utf-8')
- getjson = json.loads(output)
- ip = getjson['ip']
- country = getjson['country_name']
- lat = getjson['latitude']
- long = getjson['longitude']
- self.loc_data[data2[0]] = [ip, lat, long, country]
- self.ip_add.append(ip)
- if lat == None or long == None:
- self.barvar += 1
- self.loc_data[ip] = [ip, lat, long, country]
- with open(self.gui.e6.get(), 'w+') as outfile:
- json.dump(self.loc_data, outfile, indent=2)
- continue
- else:
- jsonobj = {"ip": ip, "hyphen": data2[1], "frank": data2[2],
- "time": data2[3], "reqline": data2[4], "HTTPcode": data2[5],
- "bytesize": data2[6], "latitude": lat, "longitude": long, "country_name": country}
- self.jsonwrite.append(jsonobj)
- except:
- self.barvar += 1
- def workerThread5(self):
- while self.running:
- if self.jsonwrite == []:
- time.sleep(0.2)
- else:
- # print('T5')
- self.barvar += 1
- writedata = self.jsonwrite.pop()
- olddata = json.load(open(self.gui.e4.get()))
- olddata.append(writedata)
- with open(self.gui.e4.get(), 'w+') as outfile:
- json.dump(olddata, outfile, indent=2)
- def parse(self):
- cwd = os.getcwd()+'/' #get work directory
- #check if txt file directory empty
- if self.gui.e0.get()=='':
- self.statusl['text'] = "insert txt log file first!";
- # check if json file empty then save to work directory as untitled.json
- if self.gui.e4.get() == '':
- self.gui.e4['state'] = 'normal'
- self.gui.e4.insert(0, cwd+'untitled.json')
- self.gui.e4['state'] = 'disabled'
- with open(self.gui.e4.get(), 'w+') as outfile:
- json.dump([], outfile)
- #if not try to load
- else:
- try:
- test = json.load(open(self.gui.e4.get()))
- except:
- self.gui.statusl['text'] = "can't open your json"
- with open(self.gui.e4.get(), 'w+') as outfile:
- json.dump([], outfile)
- #check if loc data empty
- if self.gui.e6.get() == '':
- self.gui.e6['state'] = 'normal'
- self.gui.e6.insert(0, cwd + 'locuntitled.json')
- self.gui.e6['state'] = 'disabled'
- with open(self.gui.e6.get(), 'w+') as outfile:
- json.dump({}, outfile)
- else:
- try:
- with open(self.gui.e6.get()) as f:
- self.loc_data = json.load(f)
- self.ip_add = list(self.loc_data.keys())
- except:
- self.gui.statusl['text'] = "can't open your json location data"
- self.ip_add = []
- self.loc_data = {}
- with open(self.gui.e6.get(), 'w+') as outfile:
- json.dump({}, outfile)
- try:
- with open(self.gui.e0.get(), 'rb') as f:
- text = f.read()
- self.log_data = text.splitlines()
- try:
- startindex = int(self.gui.e1.get())
- endindex = int(self.gui.e2.get())
- self.log_data = self.log_data[startindex:endindex+1]
- except:pass
- self.gui.statusl['text'] = 'start parsing...'
- self.dumbvalue=len(self.log_data)
- self.gui.mpb["maximum"] = len(self.log_data)
- self.barvar=0
- self.gui.mpb["value"] = 0
- self.thread1 = threading.Thread(target=self.workerThread1)
- self.thread2 = threading.Thread(target=self.workerThread2)
- self.thread3 = threading.Thread(target=self.workerThread3)
- self.thread4 = threading.Thread(target=self.workerThread4)
- self.thread5 = threading.Thread(target=self.workerThread5)
- self.thread1.setDaemon(1)
- self.thread2.setDaemon(1)
- self.thread3.setDaemon(1)
- self.thread4.setDaemon(1)
- self.thread5.setDaemon(1)
- self.thread1.start()
- self.thread2.start()
- self.thread3.start()
- self.thread4.start()
- self.thread5.start()
- except:
- self.gui.statusl['text'] = 'cant open your txt file :('
- def endApplication(self):
- self.running = 0
- root = Tk()
- client = ThreadedClient(root)
- root.mainloop()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement