Advertisement
Guest User

Untitled

a guest
Mar 21st, 2019
63
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 16.66 KB | None | 0 0
  1. import threading
  2. import queue
  3. from tkinter import *
  4. from tkinter import filedialog
  5. import tkinter.ttk as ttk
  6. import os
  7. import json
  8. import regex as ra
  9. import urllib.request
  10. import time
  11. class GuiPart:
  12. def __init__(self, master, queue, endCommand, parseCommand, openCommand):
  13. self.queue = queue
  14. # Set up the GUI
  15. Label(root, text="JSON PARSER :) ").grid(row=0, column=0, columnspan=8, sticky="E")
  16. Label(root, text="LOG Files.TXT").grid(row=1, column=0) # Input section row 0
  17. Label(root, text="input").grid(row=1, column=1, sticky="E")
  18. self.e0 = Entry(root, state='disabled')
  19. self.e0.grid(row=1, column=2, sticky="WE")
  20. self.b0 = Button(root, text="...", command=lambda: openCommand(0, 1)).grid(row=1, column=3, sticky="E")
  21. Label(root, text='start,stop index').grid(row=0,column=0)
  22. self.e1 = Entry(root, width=14)
  23. self.e1.grid(row=0, column=1, columnspan=2, sticky="W")
  24. self.e2 = Entry(root, width=15)
  25. self.e2.grid(row=0, column=2, columnspan=2, sticky="E")
  26. # self.e1 = Entry(root, state='disabled')
  27. # self.e1.grid(row=1, column=3, sticky="WE")
  28. # self.e2 = Entry(root, state='disabled')
  29. # self.e2.grid(row=1, column=4, sticky="WE")
  30. Label(root, text="JSON log").grid(row=3, column=4, sticky="W") # JSON log section
  31. Label(root, text="insert").grid(row=3, column=5, sticky="W") # e4,b4 for insert json e5,b5= save destination
  32. self.e4 = Entry(root, state='disabled')
  33. self.e4.grid(row=3, column=6, sticky="WE")
  34. self.b4 = Button(root, text="...", command=lambda: openCommand(0)).grid(row=3, column=7, sticky="E")
  35. Label(root, text="Location log").grid(row=3, column=0, sticky="W") # IP Location section e6,b6 for insert file, e7,b7=destination
  36. self.e6 = Entry(root, state='disabled')
  37. self.e6.grid(row=3, column=2, sticky="WE")
  38. self.b6 = Button(root, text="...", command=lambda: openCommand(1)).grid(row=3, column=3, sticky="E")
  39. Label(root, text="<=======================================================>").grid(row=5, column=0,
  40. columnspan=8, sticky="E")
  41. self.bp = Button(root, text="parse", command=parseCommand).grid(row=6, column=0, sticky="WE")
  42. self.bp = Button(root, text="stop", command=endCommand).grid(row=6, column=1,columnspan=2, sticky="W")
  43. self.numtask = Label(root, text="")
  44. self.numtask = Label(root, text="")
  45. self.numtask.grid(row=6, column=1, stick='W')
  46. failedtask = Label(root)
  47. failedtask.grid(row=6, column=2, sticky="E")
  48. self.mpb = ttk.Progressbar(root, orient="horizontal", mode="determinate")
  49. self.mpb.grid(row=6, column=3, columnspan=5, sticky="WE")
  50. Label(root, text="=======================================================").grid(row=7, column=0, columnspan=8,
  51. sticky="E")
  52. Label(root, text="STATUS:").grid(row=8, column=0, sticky="E")
  53. self.statusl = Label(root, text="")
  54. self.statusl.grid(row=8, column=1, columnspan=7)
  55. Label(root, text="=======================================================").grid(row=9, column=0, columnspan=8,
  56. sticky="E")
  57.  
  58. class ThreadedClient:
  59. """
  60. Launch the main part of the GUI and the worker thread. periodicCall and
  61. endApplication could reside in the GUI part, but putting them here
  62. means that you have all the thread controls in a single place.
  63. """
  64. def __init__(self, master):
  65. """
  66. Start the GUI and the asynchronous threads. We are in the main
  67. (original) thread of the application, which will later be used by
  68. the GUI. We spawn a new thread for the worker.
  69. """
  70. self.master = master
  71.  
  72. # Create the queue
  73. self.queue = queue.Queue()
  74.  
  75. # Set up the GUI part
  76. self.gui = GuiPart(master, self.queue, self.endApplication, self.parse, self.openfile)
  77.  
  78. # Set up the thread to do asynchronous I/O
  79. # More can be made if necessary
  80. self.key = '95a6c0b2322235de615bcd69a539e604'
  81. self.loc_data={}
  82. self.log_data = []
  83. self.ip_add = []
  84. self.jsonwrite = []
  85. self.get_ip = []
  86. self.barvar=0
  87. self.running = 1
  88. self.dumbvalue=0
  89.  
  90.  
  91. # Start the periodic call in the GUI to check if the queue contains
  92. # anything
  93. self.periodicCall()
  94. def openfile(self, loc, txt=0):
  95. # try:
  96. if txt==0:
  97. root.filename = filedialog.askopenfilename(initialdir="/home/isara/PycharmProject/cantforgether",
  98. title="Select JSON",
  99. filetypes=(("json files", "*.json"), ("all files", "*.*")))
  100. if root.filename == '':
  101. # status = Label(root, text="No file selected")
  102. self.gui.statusl['text'] = " You didn't select any json "
  103. if loc:
  104. self.gui.e6['state'] = 'normal'
  105. self.gui.e6.delete(0, END)
  106. self.gui.e6.insert(0, root.filename)
  107. self.gui.e6['state'] = 'disabled'
  108. else:
  109. self.gui.e4['state'] = 'normal'
  110. self.gui.e4.delete(0, END)
  111. self.gui.e4.insert(0, root.filename)
  112. self.gui.e4['state'] = 'disabled'
  113. else:
  114. root.filename = filedialog.askopenfilename(initialdir="/home/isara/PycharmProject/cantforgether",
  115. title="Select TXT",
  116. filetypes=(("txt files", "*.txt"), ("all files", "*.*")))
  117. if root.filename == '':
  118. self.gui.statusl['text'] = " You didn't select any txt "
  119. self.gui.e0['state'] = 'normal'
  120. self.gui.e0.delete(0, END)
  121. self.gui.e0.insert(0, root.filename)
  122. self.gui.e0['state'] = 'disabled'
  123. # except:
  124. # self.statusl['text'] = "CANNOT OPEN file :("
  125. def periodicCall(self):
  126. """
  127. Check every 100 ms if there is something new in the queue.
  128. """
  129. self.gui.mpb["value"]=self.barvar
  130. text= 'dumped data ' + str(self.barvar) + 'from total of ' + str(self.dumbvalue)
  131. self.gui.statusl['text']=text
  132. if not self.running:
  133. # This is the brutal stop of the system. You may want to do
  134. # some cleanup before actually shutting it down.
  135. time.sleep(1)
  136. import sys
  137. sys.exit(1)
  138. # root.destroy()
  139. self.master.after(100, self.periodicCall)
  140.  
  141. def workerThread1(self):
  142. while self.running:
  143. if self.log_data ==[]:
  144. time.sleep(0.2)
  145. else:
  146. data=str(self.log_data.pop())[2:-1]
  147. regex = '(.+[\s]*) (-) (.+[\s]*) \[(.*?)\] \"(.*?)\" (.+) (.+)'
  148. grouplist = ra.match(regex, data).groups()
  149. if grouplist[0] in self.ip_add:
  150. ip = self.loc_data[grouplist[0]][0]
  151. lat = self.loc_data[grouplist[0]][1]
  152. long = self.loc_data[grouplist[0]][2]
  153. country = self.loc_data[grouplist[0]][3]
  154. if lat == None or long == None:
  155. self.barvar += 1
  156. continue
  157. else:
  158. jsonobj={"ip":grouplist[0], "hyphen":grouplist[1], "frank":grouplist[2],
  159. "time":grouplist[3], "reqline":grouplist[4], "HTTPcode":grouplist[5],
  160. "bytesize":grouplist[6], "latitude":lat, "longitude":long, "country_name":country}
  161. self.jsonwrite.append(jsonobj)
  162. else:
  163. iplist=[grouplist[0],grouplist[1],grouplist[2],grouplist[3],grouplist[4],grouplist[5],grouplist[6]]
  164. self.get_ip.append(iplist)
  165.  
  166. def workerThread2(self):
  167. while self.running:
  168. if self.get_ip==[]:
  169. time.sleep(0.2)
  170. else:
  171. try:
  172. # print('T2')
  173. data2=self.get_ip.pop()
  174. url = 'http://api.ipstack.com/' + data2[0] + '?access_key=' + self.key
  175. contents = urllib.request.urlopen(url).read()
  176. output = contents.decode('utf-8')
  177. getjson = json.loads(output)
  178. ip= getjson['ip']
  179. country = getjson['country_name']
  180. lat = getjson['latitude']
  181. long = getjson['longitude']
  182. self.loc_data[data2[0]]=[ip,lat,long,country]
  183. self.ip_add.append(ip)
  184. if lat == None or long == None:
  185. self.barvar += 1
  186. self.loc_data[ip]=[ip,lat,long,country]
  187. with open(self.gui.e6.get(), 'w+') as outfile:
  188. json.dump(self.loc_data, outfile, indent=2)
  189. continue
  190. else:
  191. jsonobj = {"ip": ip, "hyphen": data2[1], "frank": data2[2],
  192. "time": data2[3], "reqline": data2[4], "HTTPcode": data2[5],
  193. "bytesize": data2[6], "latitude": lat, "longitude": long, "country_name": country}
  194. self.jsonwrite.append(jsonobj)
  195. except:self.barvar+=1
  196.  
  197. def workerThread3(self):
  198. while self.running:
  199. if self.get_ip == []:
  200. time.sleep(0.2)
  201. else:
  202. try:
  203. # print('T3')
  204. data2 = self.get_ip.pop()
  205. url = 'http://api.ipstack.com/' + data2[0] + '?access_key=' + self.key
  206. contents = urllib.request.urlopen(url).read()
  207. output = contents.decode('utf-8')
  208. getjson = json.loads(output)
  209. ip = getjson['ip']
  210. country = getjson['country_name']
  211. lat = getjson['latitude']
  212. long = getjson['longitude']
  213. self.loc_data[data2[0]] = [ip, lat, long, country]
  214. self.ip_add.append(ip)
  215. if lat == None or long == None:
  216. self.barvar += 1
  217. self.loc_data[ip] = [ip, lat, long, country]
  218. with open(self.gui.e6.get(), 'w+') as outfile:
  219. json.dump(self.loc_data, outfile, indent=2)
  220. continue
  221. else:
  222. jsonobj = {"ip": ip, "hyphen": data2[1], "frank": data2[2],
  223. "time": data2[3], "reqline": data2[4], "HTTPcode": data2[5],
  224. "bytesize": data2[6], "latitude": lat, "longitude": long, "country_name": country}
  225. self.jsonwrite.append(jsonobj)
  226. except:
  227. self.barvar += 1
  228.  
  229. def workerThread4(self):
  230. while self.running:
  231. if self.get_ip == []:
  232. time.sleep(0.2)
  233. else:
  234. try:
  235. # print('T4')
  236. data2 = self.get_ip.pop()
  237. url = 'http://api.ipstack.com/' + data2[0] + '?access_key=' + self.key
  238. contents = urllib.request.urlopen(url).read()
  239. output = contents.decode('utf-8')
  240. getjson = json.loads(output)
  241. ip = getjson['ip']
  242. country = getjson['country_name']
  243. lat = getjson['latitude']
  244. long = getjson['longitude']
  245. self.loc_data[data2[0]] = [ip, lat, long, country]
  246. self.ip_add.append(ip)
  247. if lat == None or long == None:
  248. self.barvar += 1
  249. self.loc_data[ip] = [ip, lat, long, country]
  250. with open(self.gui.e6.get(), 'w+') as outfile:
  251. json.dump(self.loc_data, outfile, indent=2)
  252. continue
  253. else:
  254. jsonobj = {"ip": ip, "hyphen": data2[1], "frank": data2[2],
  255. "time": data2[3], "reqline": data2[4], "HTTPcode": data2[5],
  256. "bytesize": data2[6], "latitude": lat, "longitude": long, "country_name": country}
  257. self.jsonwrite.append(jsonobj)
  258. except:
  259. self.barvar += 1
  260.  
  261. def workerThread5(self):
  262. while self.running:
  263. if self.jsonwrite == []:
  264. time.sleep(0.2)
  265. else:
  266. # print('T5')
  267. self.barvar += 1
  268. writedata = self.jsonwrite.pop()
  269. olddata = json.load(open(self.gui.e4.get()))
  270. olddata.append(writedata)
  271. with open(self.gui.e4.get(), 'w+') as outfile:
  272. json.dump(olddata, outfile, indent=2)
  273. def parse(self):
  274. cwd = os.getcwd()+'/' #get work directory
  275. #check if txt file directory empty
  276. if self.gui.e0.get()=='':
  277. self.statusl['text'] = "insert txt log file first!";
  278. # check if json file empty then save to work directory as untitled.json
  279. if self.gui.e4.get() == '':
  280. self.gui.e4['state'] = 'normal'
  281. self.gui.e4.insert(0, cwd+'untitled.json')
  282. self.gui.e4['state'] = 'disabled'
  283. with open(self.gui.e4.get(), 'w+') as outfile:
  284. json.dump([], outfile)
  285. #if not try to load
  286. else:
  287. try:
  288. test = json.load(open(self.gui.e4.get()))
  289. except:
  290. self.gui.statusl['text'] = "can't open your json"
  291. with open(self.gui.e4.get(), 'w+') as outfile:
  292. json.dump([], outfile)
  293. #check if loc data empty
  294. if self.gui.e6.get() == '':
  295. self.gui.e6['state'] = 'normal'
  296. self.gui.e6.insert(0, cwd + 'locuntitled.json')
  297. self.gui.e6['state'] = 'disabled'
  298. with open(self.gui.e6.get(), 'w+') as outfile:
  299. json.dump({}, outfile)
  300. else:
  301. try:
  302. with open(self.gui.e6.get()) as f:
  303. self.loc_data = json.load(f)
  304. self.ip_add = list(self.loc_data.keys())
  305. except:
  306. self.gui.statusl['text'] = "can't open your json location data"
  307. self.ip_add = []
  308. self.loc_data = {}
  309. with open(self.gui.e6.get(), 'w+') as outfile:
  310. json.dump({}, outfile)
  311. try:
  312. with open(self.gui.e0.get(), 'rb') as f:
  313. text = f.read()
  314. self.log_data = text.splitlines()
  315. try:
  316. startindex = int(self.gui.e1.get())
  317. endindex = int(self.gui.e2.get())
  318. self.log_data = self.log_data[startindex:endindex+1]
  319. except:pass
  320. self.gui.statusl['text'] = 'start parsing...'
  321. self.dumbvalue=len(self.log_data)
  322. self.gui.mpb["maximum"] = len(self.log_data)
  323. self.barvar=0
  324. self.gui.mpb["value"] = 0
  325. self.thread1 = threading.Thread(target=self.workerThread1)
  326. self.thread2 = threading.Thread(target=self.workerThread2)
  327. self.thread3 = threading.Thread(target=self.workerThread3)
  328. self.thread4 = threading.Thread(target=self.workerThread4)
  329. self.thread5 = threading.Thread(target=self.workerThread5)
  330. self.thread1.setDaemon(1)
  331. self.thread2.setDaemon(1)
  332. self.thread3.setDaemon(1)
  333. self.thread4.setDaemon(1)
  334. self.thread5.setDaemon(1)
  335. self.thread1.start()
  336. self.thread2.start()
  337. self.thread3.start()
  338. self.thread4.start()
  339. self.thread5.start()
  340. except:
  341. self.gui.statusl['text'] = 'cant open your txt file :('
  342. def endApplication(self):
  343. self.running = 0
  344.  
  345. root = Tk()
  346.  
  347. client = ThreadedClient(root)
  348. root.mainloop()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement