Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from dropbox import client, session, rest
- from dropbox.client import format_path
- import urllib2
- from StringIO import StringIO
- from Queue import Queue
- import os
- import time
- import threading
- #here is some user specific information for creating session, hence omitted
- client = client.DropboxClient(sess)
- class ChunkDownloadThread(threading.Thread):
- def __init__(self, queue, upload_id, client, download_url):
- threading.Thread.__init__(self)
- self.queue = queue
- self.upload_id = upload_id
- self.client = client
- self.download_url = download_url
- def run(self):
- while True: # not self.queue.empty():
- offset, chunk_size = self.queue.get()
- req = urllib2.Request(url=self.download_url, headers={'Range': 'bytes=%s-%s' % (offset, offset + chunk_size - 1)}) #maybe not - 1
- file_obj = urllib2.urlopen(req)
- block = file_obj.read( chunk_size )
- try:
- offset = self.client.upload_chunk( StringIO(block), chunk_size, offset, self.upload_id )[0]
- print 'Uploaded chunk with offset %s and chunk size %s' % (offset, chunk_size)
- except Exception as e:
- print 'Something went wrong when uploading chunk with offset %s and chunk size %s' % (offset, chunk_size)
- print e
- file_obj.close()
- self.queue.task_done()
- #def upload( download_url, credentials, user, target_prefix, chunk_size=4*1024*1024 ):
- # client = DropboxFile._get_client( user, DropboxFile.storage )
- def upload( download_url, client, target_prefix, chunk_size=1024*1024 ):
- def finish( path ):
- path = "/commit_chunked_upload/%s%s" % ( client.session.root, format_path(path) )
- params = dict( overwrite = False, upload_id = upload_id )
- url, params, headers = client.request( path, params, content_server=True )
- return client.rest_client.POST( url, params, headers )
- file_obj = urllib2.urlopen( download_url )
- target_length = long( file_obj.headers.dict['content-length'] )
- queue = Queue()
- name = download_url.split('/')[-1]
- offset = 0
- upload_id = None
- first_block = file_obj.read(chunk_size)
- #uploading one initial chunk to get an upload_id (the same for all subsequent chunk uploads) which is to be passed to all threads
- try:
- offset, upload_id = client.upload_chunk( StringIO(first_block), chunk_size, offset, upload_id )
- print 'First chunk uploaded'
- except Exception:
- print 'Something went wrong when uploading the first chunk'
- def main():
- #populating the queue with (offset, chunk_size) tuples
- offs = offset
- while offs < target_length:
- if target_length - offs < chunk_size:
- queue.put(( offs, target_length - offs ))
- else:
- queue.put(( offs, chunk_size ))
- offs += chunk_size
- # 10 threads take crap from the queue
- for i in range(10):
- t = ChunkDownloadThread( queue=queue, upload_id=upload_id, client=client, download_url=download_url )
- t.daemon = True
- t.start()
- if threading.active_count() == 0:
- finish( target_prefix + '/' + name )
- main()
- queue.join()
- if __name__ == '__main__':
- url = 'http://download.documentfoundation.org/libreoffice/stable/4.0.3/win/x86/LibreOffice_4.0.3_Win_x86.msi'
- upload( url, client, '/MyFolder')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement