Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- class AlreadyUploadedException(Exception):
- """This file has already been uploaded"""
- class OngoingUploadException(Exception):
- """There is an active upload"""
- class MyConcurrentUploader(ConcurrentUploader):
- def __init__(self, *args, **kwargs):
- super(MyConcurrentUploader, self).__init__(*args, **kwargs)
- def _start_upload_threads(self, result_queue, upload_id, worker_queue, filename):
- for _ in range(self._num_threads):
- thread = UploadWorkerThread(self._api, self._vault_name, filename, upload_id, worker_queue,
- result_queue, 15, 35)
- time.sleep(0.2)
- thread.start()
- self._threads.append(thread)
- class Pusher(object):
- def __init__(self, filename, description, statedb=DB_NAME):
- """Pushes (uploads) a file into glacier, first checking if it has been uploaded already"""
- self.filename = filename
- self.description = description
- self.statedb = statedb
- def uploaded(self):
- """Checks if a file has already been uploaded"""
- conn = sqlite3.connect(self.statedb)
- c = conn.cursor()
- res = c.execute('SELECT name FROM uploads')
- finished = [f[0] for f in res]
- conn.close()
- return self.description in finished
- def markuploaded(self, archiveid):
- """Marks a file as successfully uploaded"""
- conn = sqlite3.connect(self.statedb)
- c = conn.cursor()
- log.debug('INSERT INTO uploads VALUES (?,?)'.format(self.description.encode('UTF-8'), archiveid))
- c.execute('INSERT INTO uploads VALUES (?,?)', (self.description, archiveid))
- conn.commit()
- conn.close()
- return True
- def upload(self):
- """Uploads a file using a slightly modified boto ConcurrentUploader"""
- if self.uploaded():
- log.warn('{} already uploaded'.format(self.filename.encode('UTF-8')))
- raise AlreadyUploadedException()
- glacierl1 = Layer1(aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY, region_name=REGION)
- ongoing = glacierl1.list_multipart_uploads(VAULT_NAME)
- if ongoing['UploadsList']:
- raise OngoingUploadException()
- log.debug('Uploading {}'.format(self.filename.encode('UTF-8')))
- uploader = MyConcurrentUploader(glacierl1, VAULT_NAME, CHUNK_SIZE, 5)
- res = uploader.upload(self.filename, self.description)
- self.markuploaded(res)
- log.debug('Finished uploading {}'.format(self.filename.encode('UTF-8')))
- return res
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement