Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python
- '''
- module for regularly sending certain files to a remote host
- '''
- # For the transferring of files
- import pysftp
- # For filewalking, as needed for image files
- import os
- # For scheduling Tasks for midnight
- import schedule
- import time
- # For storing error info in a database
- import pymongo
- # lets us simply import our JSON config file as a module
- from jsonsempai import magic
- # config for server ip, password, etc
- import config
- def logError(error_data):
- '''
- takes data from an error and logs it into mongodb,
- error data is just a dictionary
- '''
- # sets up access to the database and sends the error data
- try:
- client = pymongo.MongoClient(config.mongo_ip, config.mongo_port)
- db = client[config.mongo_database_name]
- collection = db[config.mongo_collection_name]
- collection.insert_one(error_data)
- except:
- # if all else fails, just write to text so we dont lose info
- with open("data_dump.txt", "w+") as e:
- e.write(str(error_data))
- def copyFiles():
- '''
- Takes some xml files and sends them to the server
- and logs if there was an error
- '''
- # try to establish connection to server
- try:
- conn = pysftp.Connection(config.ip,
- username=config.username,
- password=config.password,
- port=config.port)
- ########################
- # Uploading XML Files #
- ########################
- current_directory = os.path.dirname(os.path.abspath(__file__))
- xml_files_to_copy = [
- "/Library/FileMaker\\ Server/Data/Documents/dco_fmpdsoresult.xml",
- "dco_fmpdsoresult.xml",
- "/Library/FileMaker\\ Server/Data/Documents/dco_fmpdsoresult_image_changes.xml",
- "/image-data/dco_fmpdsoresult_image_changes.xml"
- ]
- paths = [os.path.join(current_directory, fil)
- for fil in xml_files_to_copy]
- # info used if an error pops up
- for xml_file_path in paths:
- try:
- file_name = os.path.basename(xml_file_path)
- conn.put(xml_file_path)
- except OSError as e:
- logError({"description": "The file cannot be found on the local filesystem",
- "file-name": file_name,
- "file-path": xml_file_path,
- "destination-path": "/",
- "error-specifics": str(e)
- })
- except IOError as e:
- logError({"description": "The remote directory cannot be found",
- "file-name": file_name,
- "file-path": xml_file_path,
- "destination-path": "/",
- "error-specifics": str(e)
- })
- ###########################
- # Uploading Image Files #
- ###########################
- search_dir = os.path.join(
- current_directory,
- "/Volumes/appwin/ASCIMAGE/ITEMS")
- for path, subdirs, files in os.walk(search_dir):
- for name in files:
- # determine file paths by walking through directory
- absolute_file_path = os.path.join(path, name)
- file_name = os.path.basename(name)
- destination_path = os.path.join("/product-images/", file_name)
- try:
- conn.put(absolute_file_path, destination_path)
- except OSError as e:
- logError({"description": "The file cannot be found on the local filesystem",
- "file-name": file_name,
- "file-path": absolute_file_path,
- "destination-path": destination_path,
- "error-specifics": str(e)
- })
- except IOError as e:
- # The remote directory cannot be found
- logError({"description": "The remote directory cannot be found",
- "file-name": file_name,
- "file-path": absolute_file_path,
- "destination-path": destination_path,
- "error-specifics": str(e)
- })
- ########################
- # Uploading ASC Data #
- ########################
- path = os.path.join(current_directory, "/Volumes/PCFILES/EVENT")
- for path, subdirs, files in os.walk(path):
- for name in files:
- absolute_file_path = os.path.join(path, name)
- file_name = os.path.basename(name)
- try:
- # only upload files that have certain prefixes
- if "ASC_ITEM_" in file_name:
- destination_path = os.path.join(
- "/product-data/", file_name)
- conn.put(absolute_file_path, destination_path)
- elif "ASC_INVENTORY_" in file_name:
- destination_path = os.path.join(
- "/product-inventory/", file_name)
- conn.put(absolute_file_path, destination_path)
- except OSError as e:
- logError({"description": "The file cannot be found on the local filesystem",
- "file-name": file_name,
- "file-path": absolute_file_path,
- "destination-path": destination_path,
- "error-specifics": str(e)
- })
- except IOError:
- # The remote directory cannot be found
- logError({"description": "The remote directory cannot be found",
- "file-name": file_name,
- "file-path": absolute_file_path,
- "destination-path": destination_path,
- "error-specifics": str(e)
- })
- except pysftp.ConnectionException as e:
- logError({"description": "connection to server failed",
- "error-specifics": str(e)})
- # wait then try again
- time.sleep(60)
- copyFiles()
- except pysftp.CredentialException:
- raise
- except pysftp.AuthenticationException:
- raise
- except pysftp.SSHException:
- raise
- except Exception as e:
- logError({"description": "generic exception.failed to properly grab data",
- "error-specifics": str(e)})
- # wait then try again
- time.sleep(60)
- copyFiles()
- conn.close()
- def run_tasks():
- '''
- Takes all the tasks that need to be done
- (copy XML, etc) and scheadules them for midnight
- then it just waits to perform tasks
- '''
- # due to kink in module, all times are +1:00 from eastern time
- schedule.every().day.at("01:00").do(copyFiles, "Copying XML File")
- while True:
- schedule.run_pending()
- time.sleep(1)
- if __name__ == "__main__":
- run_tasks()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement