Advertisement
Guest User

Untitled

a guest
Sep 2nd, 2015
77
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 2.59 KB | None | 0 0
  1. class AlreadyUploadedException(Exception):
  2.     """This file has already been uploaded"""
  3.  
  4.  
  5. class OngoingUploadException(Exception):
  6.     """There is an active upload"""
  7.  
  8.  
  9. class MyConcurrentUploader(ConcurrentUploader):
  10.     def __init__(self, *args, **kwargs):
  11.         super(MyConcurrentUploader, self).__init__(*args, **kwargs)
  12.  
  13.     def _start_upload_threads(self, result_queue, upload_id, worker_queue, filename):
  14.         for _ in range(self._num_threads):
  15.             thread = UploadWorkerThread(self._api, self._vault_name, filename, upload_id, worker_queue,
  16.                                         result_queue, 15, 35)
  17.             time.sleep(0.2)
  18.             thread.start()
  19.             self._threads.append(thread)
  20.  
  21.  
  22. class Pusher(object):
  23.     def __init__(self, filename, description, statedb=DB_NAME):
  24.         """Pushes (uploads) a file into glacier, first checking if it has been uploaded already"""
  25.         self.filename = filename
  26.         self.description = description
  27.         self.statedb = statedb
  28.  
  29.     def uploaded(self):
  30.         """Checks if a file has already been uploaded"""
  31.         conn = sqlite3.connect(self.statedb)
  32.         c = conn.cursor()
  33.         res = c.execute('SELECT name FROM uploads')
  34.         finished = [f[0] for f in res]
  35.         conn.close()
  36.         return self.description in finished
  37.  
  38.     def markuploaded(self, archiveid):
  39.         """Marks a file as successfully uploaded"""
  40.         conn = sqlite3.connect(self.statedb)
  41.         c = conn.cursor()
  42.         log.debug('INSERT INTO uploads VALUES (?,?)'.format(self.description.encode('UTF-8'), archiveid))
  43.         c.execute('INSERT INTO uploads VALUES (?,?)', (self.description, archiveid))
  44.         conn.commit()
  45.         conn.close()
  46.         return True
  47.  
  48.     def upload(self):
  49.         """Uploads a file using a slightly modified boto ConcurrentUploader"""
  50.         if self.uploaded():
  51.             log.warn('{} already uploaded'.format(self.filename.encode('UTF-8')))
  52.             raise AlreadyUploadedException()
  53.  
  54.         glacierl1 = Layer1(aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY, region_name=REGION)
  55.         ongoing = glacierl1.list_multipart_uploads(VAULT_NAME)
  56.         if ongoing['UploadsList']:
  57.             raise OngoingUploadException()
  58.  
  59.         log.debug('Uploading {}'.format(self.filename.encode('UTF-8')))
  60.         uploader = MyConcurrentUploader(glacierl1, VAULT_NAME, CHUNK_SIZE, 5)
  61.         res = uploader.upload(self.filename, self.description)
  62.         self.markuploaded(res)
  63.         log.debug('Finished uploading {}'.format(self.filename.encode('UTF-8')))
  64.         return res
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement