Advertisement
Guest User

Untitled

a guest
Sep 14th, 2017
85
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.99 KB | None | 0 0
  1. #!/usr/bin/python
  2. import pymysql
  3. import os
  4. import time
  5. import shutil
  6. import urllib2
  7. from PIL import Image
  8. import hashlib
  9. from urlparse import urlparse
  10. from random import *
  11.  
  12.  
  13. class Process(object):
  14. "Object to process images"
  15.  
  16. def __init__(self):
  17.  
  18. self.image_folder = '/opt/www/memecdn.net/www/images'
  19. self.thumbnails = '/opt/www/memecdn.net/www/images/thumbnails'
  20. self.list = '/opt/www/memecdn.net/www/url_list.txt'
  21. self.db = 'memecdn'
  22. self.user = 'memecdn'
  23. self.sql_pass = 'bz7GY8Yp8bYwrMsX'
  24. self.sql_host = 'db.zswap.net'
  25. self.port = 3306
  26.  
  27. def sql_connect(self):
  28. '''
  29. Connects to a mysql object
  30. '''
  31. conn = pymysql.connect(host=self.sql_host,
  32. port=self.port,
  33. user=self.user,
  34. passwd=self.sql_pass,
  35. db=self.db)
  36. return conn
  37.  
  38. def remove_broken_images(self):
  39. images = next(os.walk(self.image_folder))[2]
  40. for image in images:
  41. full_path = self.image_folder + image
  42. if os.path.getsize(full_path) == 0:
  43. os.remove(full_path)
  44. print('Removed broken file: %s' % full_path)
  45.  
  46. def build_image_list(self):
  47. url_list = []
  48. with open(self.list) as f:
  49. for line in f.readlines():
  50. url_list.append(line)
  51. return url_list
  52.  
  53. def empty_table(table, self):
  54. conn = sql_connect()
  55. cur = conn.cursor()
  56. sql = "TRUNCATE TABLE %s" % (table)
  57. try:
  58. cur.execute(sql)
  59. except Exception as e:
  60. print e
  61. conn.commit()
  62.  
  63. def check_size(image, self):
  64. try:
  65. im = Image.open(image)
  66. width, height = im.size
  67. resolution = [width, height]
  68. return resolution
  69. except:
  70. return None
  71.  
  72. def create_thumbnail(image, out):
  73. size = 256, 256
  74. width = 0
  75. height = 0
  76. try:
  77. resolution = check_size(image)
  78. width = resolution[0]
  79. height = resolution[1]
  80. except:
  81. pass
  82. if width > 256 and height > 256:
  83. im = Image.open(image)
  84. im.thumbnail(size)
  85. im.save(out, "PNG")
  86.  
  87. def get_image(url):
  88. url = url.strip()
  89. image_id = int(time.time) + randint(1, 99999)
  90. image_dict = {}
  91. o = urlparse(url)
  92. # get some initial information
  93. image_dict.update({'netloc': o[1]})
  94. image_dict.update({'path': o[2]})
  95. image_dict.update({'local_file': image_id})
  96. image_dict.update({'file_type': url.split(".")[-1]})
  97. # actually get the image
  98. header = {'User-Agent': ("Mozilla/5.0 (Windows NT 6.1; "
  99. "WOW64) AppleWebKit/537.36 "
  100. "(KHTML, like Gecko) "
  101. "Chrome/43.0.2357.134 Safari/537.36")}
  102. try:
  103. req = urllib2.Request(url, headers={'User-Agent': header})
  104. raw_img = urllib2.urlopen(req).read()
  105. f = open(os.path.join('images/',
  106. str(image_dict.get('local_file')) +\
  107. "." +\
  108. image_dict.get('file_type'),
  109. 'wb'))
  110. f.write(raw_img)
  111. f.close()
  112. except Exception as e:
  113. print "failed to get %s" % (url)
  114. print e
  115. return image_dict
  116.  
  117. def clean_thumbnails(self):
  118. path = self.thumbnails
  119. shutil.rmtree(path)
  120. if not os.path.exists(path):
  121. os.makedirs(path)
  122.  
  123.  
  124. def main():
  125. '''
  126. Entry point for class Process to process images
  127. '''
  128. process_images = Process()
  129. process_images.empty_table(images)
  130. process_images.remove_broken_images()
  131. for image in process_images.build_image_list():
  132. print process_images.get_image(image)
  133.  
  134. if __name__ == '__main__':
  135. main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement