Guest User

Untitled

a guest
Oct 12th, 2018
100
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 8.04 KB | None | 0 0
  1. # imports
  2. import sys
  3. import socket
  4. import datetime
  5. import threading
  6. import signal
  7. import json
  8. import MySQLdb
  9. import S3
  10. import os.path
  11. import os
  12. import mimetypes
  13. import Image
  14.  
  15. # configuration
  16. LISTEN_HOST = ''
  17. LISTEN_PORT = 49494
  18. LISTEN_ADDR = (LISTEN_HOST,LISTEN_PORT)
  19. BUFSIZE = 4096
  20. LOG_PATH = 'logs/'
  21. ACCESS_LOG = 'access_log'
  22. ERROR_LOG = 'error_log'
  23. MYSQL_HOST = "localhost"
  24. MYSQL_PORT = 3306
  25. MYSQL_USER = "root"
  26. MYSQL_PASS = ""
  27. MYSQL_DB = "mydb"
  28. ITEM_TABLE = "attachment"
  29. S3_ACCESS_KEY = ''
  30. S3_SECRET_KEY = ''
  31. S3_BUCKET = "mybucket"
  32. S3_NEWFILE_PATH = "uploaded/"
  33. S3_IMAGE_PATH = "media/images/"
  34. S3_VIDEO_PATH = "media/images/"
  35. LOCALTMP_PATH = 'tmp/'
  36. IMAGE_RESOLUTION = {"small":"320x240","medium":"800x600","large":"1024x768"}
  37. IMAGE_EXTENSION = ".jpg"
  38. IMAGE_MIME_TYPE = "image/jpg"
  39. MAX_FIBO = 4000000 # this var is used for the max length of the fibonacci rule to calculate securekey
  40.  
  41. # variables
  42. threads = []
  43.  
  44. # time handling
  45. def current_time(string=False):
  46. time = datetime.datetime.now()
  47.  
  48. if string:
  49. time = time.replace(microsecond=0)
  50. return str(time)
  51. else:
  52. return time
  53.  
  54. # log file handling
  55. def log_error(msg):
  56. time = current_time(True)
  57. msg = msg + '\n'
  58. f = open(LOG_PATH + ERROR_LOG, 'a')
  59. f.write('%s: %s' % (time, msg))
  60. f.close()
  61.  
  62. def log_access(msg):
  63. time = current_time(True)
  64. msg = msg + '\n'
  65. f = open(LOG_PATH + ACCESS_LOG, 'a')
  66. f.write('%s: %s' % (time, msg))
  67. f.close()
  68.  
  69. # listen tcp server
  70. try:
  71. s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  72. except socket.error, msg:
  73. log_error(msg(1))
  74. sys.exit(1)
  75.  
  76.  
  77. # init mimetypes
  78. mimetypes.init()
  79.  
  80. # cleans up when sigint or when script ends
  81. def cleanup(*args):
  82. print 'Exiting'
  83.  
  84. global threads
  85. global s
  86.  
  87. # Close the server socket
  88. s.close()
  89. s = None
  90.  
  91. # Wait for all threads
  92. for t in threads:
  93. t.join()
  94.  
  95. sys.exit(0)
  96.  
  97. def getFileExtension(file):
  98. ext = os.path.splitext(file)[1]
  99. return ext
  100.  
  101. # handles the socket connections
  102. class ConnectionHandler(threading.Thread):
  103. def __init__(self,(conn,addr)):
  104. self.conn = conn
  105. self.addr = addr
  106. self.s3conn = ''
  107. self.secretkey = ''
  108. self.articleid = ''
  109. self.nrimg = ''
  110. self.itemlist = ''
  111. self.authenticated = False
  112.  
  113. # connect mysql server
  114. try:
  115. self.db = MySQLdb.connect(MYSQL_HOST, MYSQL_USER, MYSQL_PASS, MYSQL_DB)
  116. self.dbcursor = self.db.cursor()
  117. except Exception, e:
  118. raise e
  119.  
  120. threading.Thread.__init__(self)
  121.  
  122. def run(self):
  123. log_access('Connection from ' + str(self.addr))
  124.  
  125. while self.conn:
  126. data = self.conn.recv(BUFSIZE)
  127. if data:
  128. # encode json data received and validate it
  129. try:
  130. data = json.loads(data)
  131. except ValueError, msg:
  132. log_error(str(self.addr) + ": " + str(msg))
  133. break
  134.  
  135. for key,value in data.iteritems():
  136. log_access('Received: ' + str(key) + " -> " + str(value))
  137.  
  138. if key == "articleid":
  139. self.articleid = value
  140. elif key == "nrimg":
  141. self.nrimg = value
  142. elif key == "secretkey":
  143. self.secretkey = value
  144. else:
  145. pass
  146.  
  147. # check that articleid and nrimg got data
  148. if self.articleid and self.nrimg:
  149. newkey = self.checkKey()
  150. else:
  151. log_error(str(self.addr) + ": could not generate secretkey")
  152. break
  153.  
  154. if newkey == self.secretkey:
  155. log_access(str(self.addr) + " authentication successful")
  156. self.authenticated = True
  157. else:
  158. log_error("Error Authenticating " + str(self.addr) + ". Key was " + str(self.secretkey) + " but needs to be " + str(newkey))
  159. break
  160.  
  161. if self.authenticated:
  162. # query DB for image items
  163. # connect to database and receive media info
  164. try:
  165. self.itemlist = self.queryItems(self.articleid)
  166. except Exception, e:
  167. raise e
  168.  
  169. if self.itemlist:
  170. print str(self.itemlist)
  171. # connect to S3 and get images
  172. self.connectS3()
  173. else:
  174. log_error("Imagelist is empty!")
  175.  
  176. # convert them
  177. # update DB
  178.  
  179. # close connections
  180.  
  181. self.dbcursor.close()
  182. self.db.close()
  183.  
  184. pass
  185.  
  186. else:
  187. log_access('Connection closed by ' + str(self.addr))
  188. break
  189. self.conn.close()
  190. self.conn = None
  191.  
  192. def queryItems(self, id):
  193. itemlist = []
  194. rows = ''
  195.  
  196. try:
  197. self.dbcursor.execute("SELECT * from " + ITEM_TABLE)
  198. rows = self.dbcursor.fetchall()
  199. except MySQLdb.ProgrammingError, e:
  200. log_error(str(e))
  201.  
  202. if rows:
  203. for row in rows:
  204. itemlist.append(row[1])
  205. else:
  206. log_error("Database result for current query is empty!")
  207.  
  208. return itemlist
  209.  
  210. def checkKey(self):
  211. z = [self.articleid,self.nrimg]
  212.  
  213. while z[-1] < MAX_FIBO:
  214. z.append(z[-1] + z[-2])
  215.  
  216. return sum([x for x in z if x % 2 == 0])
  217.  
  218. def connectS3(self):
  219. self.s3conn = S3.AWSAuthConnection(S3_ACCESS_KEY, S3_SECRET_KEY)
  220. # get bucket list
  221. if (self.s3conn.check_bucket_exists(S3_BUCKET).status == 200):
  222. print '---- bucket status OK ----'
  223. for item in self.itemlist:
  224. log_access('--- handling item %s ---' % item)
  225. self.handleS3Item(item)
  226. else:
  227. print '---- bucket doesnt exist ----'
  228.  
  229. def handleS3Item(self,item):
  230. itempath = S3_NEWFILE_PATH + item
  231. localitem = LOCALTMP_PATH + item
  232. itemext = getFileExtension(item)
  233.  
  234. try:
  235. self.getS3Item(itempath,localitem)
  236. except Exception, e:
  237. log_error(str(e))
  238.  
  239. try:
  240. filetype = self.handleItemType(itemext)
  241. except Exception, e:
  242. log_error(str(e))
  243.  
  244. if filetype:
  245. if filetype == "image":
  246. self.handleImage(localitem)
  247. elif filetype == "video":
  248. self.handleVideo()
  249.  
  250. os.unlink(localitem)
  251.  
  252. def getS3Item(self,remoteitem,localitem):
  253. try:
  254. itemdata = self.s3conn.get(S3_BUCKET, remoteitem).object.data
  255. except Exception, e:
  256. log_error(str(e))
  257.  
  258. if itemdata:
  259. f = open(localitem, 'wb+')
  260. f.write(itemdata)
  261. f.close()
  262. else:
  263. log_error("S3 Object doesnt have data!")
  264.  
  265. log_access('--- done getting item %s ---' % localitem)
  266.  
  267. def handleItemType(self,itemext):
  268. try:
  269. mime = mimetypes.types_map[itemext]
  270. except Exception, e:
  271. log_error(str(e))
  272.  
  273. if "image" in mime:
  274. return "image"
  275. elif "video" in mime:
  276. return "video"
  277. else:
  278. pass
  279.  
  280. def handleImage(self, imagefile):
  281. log_access('--- opened image %s ---' % imagefile)
  282. try:
  283. self.convertImage(imagefile)
  284. except Exception, e:
  285. raise e
  286.  
  287.  
  288. def convertImage(self, imagefile):
  289.  
  290. imagedata = Image.open(imagefile)
  291.  
  292. for k,v in IMAGE_RESOLUTION.iteritems():
  293. resizename = k
  294. resolution = v.split('x')
  295. width = int(resolution[0])
  296. height = int(resolution[1])
  297.  
  298. try:
  299. log_access('--- resizing image %s ---' % imagefile)
  300. img = imagedata.resize((width,height), Image.NEAREST)
  301. except Exception, e:
  302. log_error(e)
  303.  
  304. newname = self.generateNewImageName(imagefile, resizename)
  305.  
  306. try:
  307. log_access('--- trying to save new image %s ---' % newname)
  308. img.save(newname)
  309. except Exception, e:
  310. log_error(e)
  311.  
  312. log_access('--- new image created %s ---' % newname)
  313.  
  314. try:
  315. self.putS3Item(newname,IMAGE_MIME_TYPE)
  316. except Exception, e:
  317. log_error(e)
  318.  
  319. def generateNewImageName(self,imgfile,addname):
  320. path, ext = os.path.splitext(imgfile)
  321.  
  322. newname = path + "_" + addname + ext
  323.  
  324. return newname
  325.  
  326. def handleVideo(self):
  327. pass
  328.  
  329.  
  330. def putS3Item(self, item, contenttype):
  331. itemreplace = item.replace(LOCALTMP_PATH,'')
  332. s3itempath = S3_IMAGE_PATH + itemreplace
  333. itemdata = open(item, 'rb').read()
  334.  
  335. try:
  336. log_access('--- trying to upload %s ---' % itemreplace)
  337. self.s3conn.put(S3_BUCKET,s3itempath, S3.S3Object(itemdata),
  338. {'x-amz-acl': 'public-read', 'Content-Type': contenttype})
  339. except Exception, e:
  340. log_error(e)
  341.  
  342. self.delTmpItem(item)
  343.  
  344. def delTmpItem(self,item):
  345. try:
  346. os.unlink(item)
  347. except Exception, e:
  348. raise e
  349.  
  350. log_access("deleted temp file %s " % item)
  351.  
  352. # runs the actual tcp server
  353. try:
  354. s.bind((LISTEN_HOST, LISTEN_PORT))
  355. s.listen(5)
  356. log_access('Server started!')
  357. except socket.error, e:
  358. s.close()
  359.  
  360. if s is None:
  361. log_error('could not open socket')
  362. sys.exit(1)
  363.  
  364. # Catch some signals
  365. signal.signal(signal.SIGINT, cleanup)
  366. signal.signal(signal.SIGTERM, cleanup)
  367.  
  368. # builds up the single client connections
  369. # handle incoming connections
  370. while s:
  371. print "Threads:", len(threads)
  372. ch = ConnectionHandler(s.accept())
  373. ch.daemon = True
  374. ch.start()
  375. threads.append(ch)
  376.  
  377. cleanup()
Add Comment
Please, Sign In to add comment