Advertisement
Guest User

Untitled

a guest
Sep 14th, 2017
62
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.51 KB | None | 0 0
  1. #!/usr/bin/python
  2. import pymysql
  3. import os
  4. import time
  5. import shutil
  6. import urllib2
  7. from PIL import Image
  8. import hashlib
  9. from urlparse import urlparse
  10. from random import *
  11.  
  12. class Process(object):
  13. "Object to process images"
  14.  
  15. def __init__(self):
  16.  
  17. self.image_folder = '/opt/www/memecdn.net/www/images'
  18. self.thumbnails = '/opt/www/memecdn.net/www/images/thumbnails'
  19. self.list = '/opt/www/memecdn.net/www/url_list.txt'
  20. self.db = 'memecdn'
  21. self.user = 'memecdn'
  22. self.sql_pass = 'bz7GY8Yp8bYwrMsX'
  23. self.sql_host = 'db.zswap.net'
  24. self.port = 3306
  25.  
  26. def sql_connect(self):
  27. '''
  28. Connects to a mysql object
  29. '''
  30. conn = pymysql.connect(host=self.sql_host, port=self.port, user=self.user, passwd=self.sql_pass, db=self.db)
  31. return conn
  32.  
  33. def remove_broken_images(self):
  34. images = next(os.walk(self.image_folder))[2]
  35. for image in images:
  36. full_path = self.image_folder + image
  37. if os.path.getsize(full_path) == 0:
  38. os.remove(full_path)
  39. print('Removed broken file: %s' % full_path)
  40.  
  41. def build_image_list(self):
  42. url_list = []
  43. with open(self.list) as f:
  44. for line in f.readlines():
  45. url_list.append(line)
  46. return url_list
  47.  
  48. def empty_table(table, self):
  49. conn = sql_connect()
  50. cur = conn.cursor()
  51. sql = "TRUNCATE TABLE %s" % (table)
  52. try:
  53. cur.execute(sql)
  54. except Exception as e:
  55. print e
  56. conn.commit()
  57.  
  58. def check_size(image, self):
  59. try:
  60. im = Image.open(image)
  61. width, height = im.size
  62. resolution = [width, height]
  63. return resolution
  64. except:
  65. return None
  66.  
  67. def create_thumbnail(image, out):
  68. size = 256, 256
  69. width = 0
  70. height = 0
  71. try:
  72. resolution = check_size(image)
  73. width = resolution[0]
  74. height = resolution[1]
  75. except:
  76. pass
  77. if width > 256 and height > 256:
  78. im = Image.open(image)
  79. im.thumbnail(size)
  80. im.save(out, "PNG")
  81.  
  82. def get_image(url):
  83. url = url.strip()
  84. image_id = int(time.time)) + randint(1,99999)
  85. image_dict = {}
  86. o = urlparse(url)
  87. # get some initial information
  88. image_dict.update({'netloc': o[1]})
  89. image_dict.update({'path': o[2]})
  90. image_dict.update({'local_file': image_id})
  91. image_dict.update({'file_type': url.split(".")[-1]})
  92. # actually get the image
  93. header={'User-Agent':"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36"}
  94. try:
  95. req = urllib2.Request(url, headers={'User-Agent' : header })
  96. raw_img = urllib2.urlopen(req).read()
  97. f = open(os.path.join('images/', str(image_dict.get('local_file')) + "." + image_dict.get('file_type')), 'wb')
  98. f.write(raw_img)
  99. f.close()
  100. except Exception as e:
  101. print "failed to get %s" % (url)
  102. print e
  103. return image_dict
  104.  
  105. def clean_thumbnails(self):
  106. path = self.thumbnails
  107. shutil.rmtree(path)
  108. if not os.path.exists(path):
  109. os.makedirs(path)
  110.  
  111.  
  112. def main():
  113. '''
  114. Entry point for class Process to process images
  115. '''
  116. process_images = Process()
  117. process_images.empty_table(images)
  118. process_images.remove_broken_images()
  119. for image in process_images.build_image_list():
  120. print process_images.get_image(image)
  121.  
  122. if __name__ = '__main__':
  123. main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement