Advertisement
Guest User

Untitled

a guest
Feb 18th, 2016
103
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.14 KB | None | 0 0
  1. #!/usr/bin/env python
  2. '''
  3. module for regularly sending certain files to a remote host
  4. '''
  5.  
  6. # For the transferring of files
  7. import pysftp
  8.  
  9. # For filewalking, as needed for image files
  10. import os
  11.  
  12. # For scheduling Tasks for midnight
  13. import schedule
  14. import time
  15.  
  16. # For storing error info in a database
  17. import pymongo
  18.  
  19. # lets us simply import our JSON config file as a module
  20. from jsonsempai import magic
  21.  
  22. # config for server ip, password, etc
  23. import config
  24.  
  25.  
  26. def logError(error_data):
  27. '''
  28. takes data from an error and logs it into mongodb,
  29.  
  30. error data is just a dictionary
  31. '''
  32.  
  33. # sets up access to the database and sends the error data
  34. try:
  35. client = pymongo.MongoClient(config.mongo_ip, config.mongo_port)
  36. db = client[config.mongo_database_name]
  37. collection = db[config.mongo_collection_name]
  38.  
  39. collection.insert_one(error_data)
  40.  
  41. except:
  42. # if all else fails, just write to text so we dont lose info
  43. with open("data_dump.txt", "w+") as e:
  44. e.write(str(error_data))
  45.  
  46.  
  47. def copyFiles():
  48. '''
  49. Takes some xml files and sends them to the server
  50. and logs if there was an error
  51. '''
  52. # try to establish connection to server
  53. try:
  54. conn = pysftp.Connection(config.ip,
  55. username=config.username,
  56. password=config.password,
  57. port=config.port)
  58.  
  59. ########################
  60. # Uploading XML Files #
  61. ########################
  62.  
  63. current_directory = os.path.dirname(os.path.abspath(__file__))
  64.  
  65. xml_files_to_copy = [
  66. "/Library/FileMaker\\ Server/Data/Documents/dco_fmpdsoresult.xml",
  67. "dco_fmpdsoresult.xml",
  68. "/Library/FileMaker\\ Server/Data/Documents/dco_fmpdsoresult_image_changes.xml",
  69. "/image-data/dco_fmpdsoresult_image_changes.xml"
  70. ]
  71.  
  72. paths = [os.path.join(current_directory, fil)
  73. for fil in xml_files_to_copy]
  74.  
  75. # info used if an error pops up
  76.  
  77. for xml_file_path in paths:
  78. try:
  79. file_name = os.path.basename(xml_file_path)
  80. conn.put(xml_file_path)
  81.  
  82. except OSError as e:
  83. logError({"description": "The file cannot be found on the local filesystem",
  84. "file-name": file_name,
  85. "file-path": xml_file_path,
  86. "destination-path": "/",
  87. "error-specifics": str(e)
  88. })
  89. except IOError as e:
  90. logError({"description": "The remote directory cannot be found",
  91. "file-name": file_name,
  92. "file-path": xml_file_path,
  93. "destination-path": "/",
  94. "error-specifics": str(e)
  95. })
  96.  
  97. ###########################
  98. # Uploading Image Files #
  99. ###########################
  100.  
  101. search_dir = os.path.join(
  102. current_directory,
  103. "/Volumes/appwin/ASCIMAGE/ITEMS")
  104.  
  105. for path, subdirs, files in os.walk(search_dir):
  106. for name in files:
  107. # determine file paths by walking through directory
  108. absolute_file_path = os.path.join(path, name)
  109. file_name = os.path.basename(name)
  110. destination_path = os.path.join("/product-images/", file_name)
  111.  
  112. try:
  113. conn.put(absolute_file_path, destination_path)
  114.  
  115. except OSError as e:
  116. logError({"description": "The file cannot be found on the local filesystem",
  117. "file-name": file_name,
  118. "file-path": absolute_file_path,
  119. "destination-path": destination_path,
  120. "error-specifics": str(e)
  121. })
  122.  
  123. except IOError as e:
  124. # The remote directory cannot be found
  125. logError({"description": "The remote directory cannot be found",
  126. "file-name": file_name,
  127. "file-path": absolute_file_path,
  128. "destination-path": destination_path,
  129. "error-specifics": str(e)
  130. })
  131.  
  132. ########################
  133. # Uploading ASC Data #
  134. ########################
  135.  
  136. path = os.path.join(current_directory, "/Volumes/PCFILES/EVENT")
  137.  
  138. for path, subdirs, files in os.walk(path):
  139. for name in files:
  140. absolute_file_path = os.path.join(path, name)
  141. file_name = os.path.basename(name)
  142.  
  143. try:
  144. # only upload files that have certain prefixes
  145. if "ASC_ITEM_" in file_name:
  146. destination_path = os.path.join(
  147. "/product-data/", file_name)
  148. conn.put(absolute_file_path, destination_path)
  149.  
  150. elif "ASC_INVENTORY_" in file_name:
  151. destination_path = os.path.join(
  152. "/product-inventory/", file_name)
  153.  
  154. conn.put(absolute_file_path, destination_path)
  155.  
  156. except OSError as e:
  157. logError({"description": "The file cannot be found on the local filesystem",
  158. "file-name": file_name,
  159. "file-path": absolute_file_path,
  160. "destination-path": destination_path,
  161. "error-specifics": str(e)
  162. })
  163. except IOError:
  164. # The remote directory cannot be found
  165. logError({"description": "The remote directory cannot be found",
  166. "file-name": file_name,
  167. "file-path": absolute_file_path,
  168. "destination-path": destination_path,
  169. "error-specifics": str(e)
  170. })
  171.  
  172. except pysftp.ConnectionException as e:
  173. logError({"description": "connection to server failed",
  174. "error-specifics": str(e)})
  175.  
  176. # wait then try again
  177. time.sleep(60)
  178. copyFiles()
  179.  
  180. except pysftp.CredentialException:
  181. raise
  182. except pysftp.AuthenticationException:
  183. raise
  184. except pysftp.SSHException:
  185. raise
  186. except Exception as e:
  187. logError({"description": "generic exception.failed to properly grab data",
  188. "error-specifics": str(e)})
  189. # wait then try again
  190. time.sleep(60)
  191. copyFiles()
  192.  
  193. conn.close()
  194.  
  195.  
  196. def run_tasks():
  197. '''
  198. Takes all the tasks that need to be done
  199. (copy XML, etc) and scheadules them for midnight
  200.  
  201. then it just waits to perform tasks
  202. '''
  203.  
  204. # due to kink in module, all times are +1:00 from eastern time
  205. schedule.every().day.at("01:00").do(copyFiles, "Copying XML File")
  206.  
  207. while True:
  208. schedule.run_pending()
  209. time.sleep(1)
  210.  
  211. if __name__ == "__main__":
  212. run_tasks()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement