Guest User

Untitled

a guest
Sep 24th, 2018
102
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.72 KB | None | 0 0
  1. import base64
  2. import hashlib
  3. import mimetypes
  4. import os
  5. import threading
  6.  
  7. from boto.s3.key import Key
  8. from django.conf import settings
  9. from django.core.files.storage import get_storage_class
  10. from storages.backends.s3boto import S3BotoStorage
  11.  
  12. STATIC_LOCAL = getattr(settings, 'STATIC_LOCAL', settings.DEBUG)
  13.  
  14. _threadlocals = threading.local()
  15.  
  16. def checksum(data):
  17. pos = data.tell()
  18. data.seek(0)
  19. checksum = hashlib.md5(data.read())
  20. hexdigest = checksum.hexdigest()
  21. b64digest = base64.b64encode(checksum.digest())
  22. data.seek(pos)
  23. return (hexdigest, b64digest)
  24.  
  25. class MediaSyncMiddleware(object):
  26.  
  27. def process_request(self, request):
  28. enc = request.META.get('HTTP_ACCEPT_ENCODING', '')
  29. _threadlocals.client_supports_gzip = 'gzip' in enc
  30.  
  31. class MediaSyncStorage(S3BotoStorage):
  32.  
  33. def _get_key(self, name):
  34. k = self.bucket.get_key(self._encode_name(name))
  35. if not k:
  36. k = self.bucket.new_key(self._encode_name(name))
  37. return k
  38.  
  39. def _save(self, name, content):
  40.  
  41. cleaned_name = self._clean_name(name)
  42. name = self._normalize_name(cleaned_name)
  43. content.name = cleaned_name
  44.  
  45. content_type = getattr(content, 'content_type', mimetypes.guess_type(name)[0] or Key.DefaultContentType)
  46.  
  47. (hexdigest, b64digest) = checksum(content)
  48.  
  49. headers = self.headers.copy()
  50. headers['Content-Type'] = content_type
  51.  
  52. # sync normal file
  53.  
  54. k = self._get_key(name)
  55.  
  56. s3digest = (k.get_metadata('mediasync-checksum') or '').replace(' ', '+')
  57.  
  58. print s3digest, ':', b64digest, ':', s3digest == b64digest
  59.  
  60. if s3digest != b64digest:
  61.  
  62. k.set_metadata('mediasync-checksum', b64digest)
  63. k.set_contents_from_file(content, headers=headers, policy=self.acl,
  64. reduced_redundancy=self.reduced_redundancy,
  65. md5=(hexdigest, b64digest))
  66.  
  67. # sync gzipped file
  68.  
  69. if self.gzip and content_type in self.gzip_content_types:
  70.  
  71. name = '%s.gz' % name
  72.  
  73. k = self._get_key(name)
  74.  
  75. content = self._compress_content(content)
  76. headers['Content-Encoding'] = 'gzip'
  77. headers["Content-Disposition"] = 'inline; filename="%sgz"' % name.split('/')[-1]
  78.  
  79. k.set_contents_from_file(content, headers=headers, policy=self.acl,
  80. reduced_redundancy=self.reduced_redundancy)
  81.  
  82. return cleaned_name
  83.  
  84. def url(self, name):
  85.  
  86. if STATIC_LOCAL:
  87. return os.path.join(settings.STATIC_URL, name)
  88.  
  89. name = self._normalize_name(self._clean_name(name))
  90. content_type = mimetypes.guess_type(name)[0] or Key.DefaultContentType
  91.  
  92. if self.gzip and content_type in self.gzip_content_types:
  93. if _threadlocals.client_supports_gzip:
  94. name = '%s.gz' % name
  95.  
  96. if self.custom_domain:
  97. return "%s://%s/%s" % ('https' if self.secure_urls else 'http', self.custom_domain, name)
  98. else:
  99. return self.connection.generate_url(self.querystring_expire, method='GET', \
  100. bucket=self.bucket.name, key=self._encode_name(name), query_auth=self.querystring_auth, \
  101. force_http=not self.secure_urls)
  102.  
  103. class CachedMediaSyncStorage(MediaSyncStorage):
  104. """
  105. S3 storage backend that saves the files locally, too.
  106. """
  107. def __init__(self, *args, **kwargs):
  108. super(CachedMediaSyncStorage, self).__init__(*args, **kwargs)
  109. self.local_storage = get_storage_class(
  110. "compressor.storage.CompressorFileStorage")()
  111.  
  112. def save(self, name, content):
  113. self.local_storage._save(name, content)
  114. name = super(CachedMediaSyncStorage, self).save(name, content)
  115. return name
Add Comment
Please, Sign In to add comment