SHARE
TWEET

Untitled

a guest May 21st, 2017 55 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. # coding:utf-8
  2. """ 下载任务监控与文件上传
  3. Detect the specified-type files in the folder which finished downloading and transfer them to FTP server.
  4. """
  5. # ----------------------------
  6. # Author: Eli, Kun Liu        
  7. # Start date: 2017-04
  8. # Latest edit: 2017-05-22
  9. # -----------------------------
  10.  
  11. __python__ = 2.7
  12.  
  13. import os
  14. import threading
  15. import shutil
  16. import sys
  17. import time
  18.  
  19. from ftplib import FTP
  20.  
  21.  
  22. ISOTIMEFORMAT = '%Y-%m-%d %X'
  23. sizeWritten = 0
  24. totalSize = 0
  25. TARGET_DIR_NAME = settings.TARGET_DIR_NAME
  26. THREAD_NUM = 4
  27. UPLOAD_BUFSIZE = 8192
  28.  
  29. # 根据后缀筛选文件
  30. legal_url_extensions = ('.dmg', '.pkg', '.sit', '.hqx', '.sitx', '.dmg.gz', '.dmg.sit', '.zip')
  31.  
  32. def ftp_login():
  33.     ftp_host_address = settings.ADDRESS
  34.     ftp_port = settings.PORT
  35.     ftp_username = settings.USERNAME
  36.     ftp_pass = settings.PASSWORDS
  37.     ftp = FTP()
  38.     ftp.set_debuglevel(0)
  39.     ftp.connect(ftp_host_address, ftp_port)
  40.     ftp.login(ftp_username, ftp_pass)
  41.     try:
  42.         ftp.cwd('/%s'%TARGET_DIR_NAME)
  43.     except Exception, ex:
  44.         print "[ERROR] " + str(ex)
  45.         sys.exit()
  46.     return ftp
  47.  
  48.  
  49. def upload_callback(buf):
  50.     global sizeWritten
  51.     global totalSize
  52.     sizeWritten += len(buf)
  53.     status = r"%10d [%3.2f%%]" % (sizeWritten, sizeWritten * 100.0 / totalSize)
  54.     print status,
  55.     print (chr(8)) * (len(status) + 2),
  56.  
  57.  
  58. def get_current_dir_filenames(ftp):
  59.     lines = []
  60.     ftp.dir(lines.append)
  61.     filenames = []
  62.     for line in lines:
  63.         part = line.split(' ')[-1]
  64.         filenames.append(part)
  65.     return filenames
  66.  
  67.  
  68. def upload_to_ftp(local_file):
  69.     global sizeWritten
  70.     global totalSize
  71.     ftp = ftp_login()
  72.     print "****************************New Start**********************************"
  73.     print time.strftime(ISOTIMEFORMAT, time.localtime())
  74.     print "Uploading %s to Ftp. Please wait......."%local_file.split("/")[-1]
  75.     bufsize = UPLOAD_BUFSIZE
  76.     if not os.path.isfile(local_file):
  77.         return
  78.     file_handler = open(local_file, "rb")
  79.     totalSize += os.path.getsize(local_file)
  80.     sizeWritten = 0
  81.     ftp.storbinary('STOR %s' % os.path.basename(local_file), file_handler, bufsize, callback=upload_callback)
  82.     file_handler.close()
  83.     print "\t[Success]" + local_file + " Upload into FTP Success!"
  84.     ftp.quit()
  85.  
  86.  
  87. def extract_file(local_dir):
  88.     for root, dirs, files in os.walk(local_dir):
  89.         for f in files:
  90.             if f.endswith(legal_url_extensions):
  91.                 file_path = os.path.join(root, f)
  92.                 try:
  93.                     upload_to_ftp(file_path)
  94.                 except Exception, ex:
  95.                     print "[ERROR in UPLOAD] " + str(ex)
  96.                 else:
  97.                     os.remove(file_path)
  98.                     print "delete local files"
  99.     shutil.rmtree(local_dir)
  100.  
  101.  
  102. def upload_files_dirs(local_dir):
  103.     if os.path.isdir(local_dir):  # if it is dir , upload it in another program .
  104.         extract_file(local_dir)
  105.         return
  106.     try:
  107.         if local_dir.endswith(legal_url_extensions):
  108.             upload_to_ftp(local_dir)
  109.     except Exception, ex:
  110.         print "[ERROR in UPLOAD] " + str(ex)
  111.     else:
  112.         os.remove(local_dir)
  113.         print "delete local files"
  114.  
  115.  
  116. def monitor_file_from(dir):
  117.     allfiles = os.listdir(dir)
  118.     # 仅对 aria2c 已完成下载的文件或文件夹内容进行上传
  119.     for allfile in allfiles:
  120.         if allfile.endswith('aria2') or allfile.endswith('temp'):
  121.             continue
  122.         tmp_1 = allfile + '.aria2'
  123.         tmp_2 = allfile + '.aria2__temp'
  124.         if tmp_1 in allfiles or tmp_2 in allfiles:
  125.             continue
  126.         local_dir = dir + allfile
  127.         upload_files_dirs(local_dir)
  128.  
  129.  
  130. def timer_task():
  131.     print "**************************Enter in Timer task ************************** "
  132.     threads_list = []
  133.     dir_list = ['/root/remote_transfor/']
  134.     for dir in dir_list:
  135.         monitor_file_from(dir)
  136.     print "**************************Get out Timer task ************************** "
  137.     global t
  138.     t = threading.Timer(10, timer_task)
  139.     t.start()
  140.  
  141.  
  142. t = threading.Timer(10, timer_task)
  143.  
  144.  
  145. def main():
  146.     global t
  147.     t.start()
  148.  
  149.  
  150. if __name__ == '__main__':
  151.     main()
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top