Advertisement
Guest User

dropbox multithreaded upload

a guest
May 22nd, 2013
302
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 3.47 KB | None | 0 0
  1. from dropbox import client, session, rest
  2. from dropbox.client import format_path
  3. import urllib2
  4. from StringIO import StringIO
  5. from Queue import Queue
  6. import os
  7. import time
  8. import threading
  9.  
  10. #here is some user specific information for creating session, hence omitted
  11.  
  12. client = client.DropboxClient(sess)
  13.  
  14.  
  15. class ChunkDownloadThread(threading.Thread):
  16.     def __init__(self, queue, upload_id, client, download_url):
  17.         threading.Thread.__init__(self)
  18.         self.queue = queue
  19.         self.upload_id = upload_id
  20.         self.client = client
  21.         self.download_url = download_url
  22.  
  23.     def run(self):
  24.         while True: # not self.queue.empty():
  25.             offset, chunk_size = self.queue.get()
  26.             req = urllib2.Request(url=self.download_url, headers={'Range': 'bytes=%s-%s' % (offset, offset + chunk_size - 1)}) #maybe not - 1
  27.             file_obj = urllib2.urlopen(req)
  28.             block = file_obj.read( chunk_size )
  29.             try:
  30.                 offset = self.client.upload_chunk( StringIO(block), chunk_size, offset, self.upload_id )[0]
  31.                 print 'Uploaded chunk with offset %s and chunk size %s' % (offset, chunk_size)
  32.             except Exception as e:
  33.                 print 'Something went wrong when uploading chunk with offset %s and chunk size %s' % (offset, chunk_size)
  34.                 print e
  35.             file_obj.close()
  36.             self.queue.task_done()
  37.  
  38.  
  39. #def upload( download_url, credentials, user, target_prefix, chunk_size=4*1024*1024 ):
  40. #    client = DropboxFile._get_client( user, DropboxFile.storage )
  41. def upload( download_url, client, target_prefix, chunk_size=1024*1024 ):
  42.  
  43.     def finish( path ):
  44.         path = "/commit_chunked_upload/%s%s" % ( client.session.root, format_path(path) )
  45.         params = dict( overwrite = False, upload_id = upload_id )
  46.         url, params, headers = client.request( path, params, content_server=True )
  47.         return client.rest_client.POST( url, params, headers )
  48.  
  49.     file_obj = urllib2.urlopen( download_url )
  50.     target_length = long( file_obj.headers.dict['content-length'] )
  51.     queue = Queue()
  52.     name = download_url.split('/')[-1]
  53.     offset = 0
  54.     upload_id = None
  55.     first_block = file_obj.read(chunk_size)
  56.     #uploading one initial chunk to get an upload_id (the same for all subsequent chunk uploads) which is to be passed to all threads
  57.     try:
  58.         offset, upload_id = client.upload_chunk( StringIO(first_block), chunk_size, offset, upload_id )
  59.         print 'First chunk uploaded'
  60.     except Exception:
  61.         print 'Something went wrong when uploading the first chunk'
  62.  
  63.     def main():
  64.         #populating the queue with (offset, chunk_size) tuples
  65.         offs = offset
  66.         while offs < target_length:
  67.             if target_length - offs < chunk_size:
  68.                 queue.put(( offs, target_length - offs ))
  69.             else:
  70.                 queue.put(( offs, chunk_size  ))
  71.             offs += chunk_size
  72.         # 10 threads take crap from the queue
  73.         for i in range(10):
  74.             t = ChunkDownloadThread( queue=queue, upload_id=upload_id, client=client, download_url=download_url )
  75.             t.daemon = True
  76.             t.start()
  77.         if threading.active_count() == 0:
  78.             finish( target_prefix + '/' + name )
  79.  
  80.     main()
  81.     queue.join()
  82.  
  83.  
  84. if __name__ == '__main__':
  85.     url = 'http://download.documentfoundation.org/libreoffice/stable/4.0.3/win/x86/LibreOffice_4.0.3_Win_x86.msi'
  86.     upload( url, client, '/MyFolder')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement