Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python3
- ##
- # @file filesync.py
- # @author redxef
- # @version 0.0.0-r0
- # @since 2018-10-22
- #
- # @brief A simple sync utility for sftp servers.
- # This utility is meant to be run in the background and called periodically and
- # synchronizes folders on an sftp server with the local folders specified in the config.
- # Multiple folders (or files) can be selected and will then be synced with the server.
- # This utility is only really useful with multiple devices for syncing local folders.
- # Prime examples would be the Documents folder.
- ##
- import sys
- import os
- import socket
- import pathlib
- import configparser
- import pysftp
- import getopt
- import paramiko
- ##
- # Stores the configuration and creates a default configuration file in case there is none.
- class FilesyncConf(object):
- def __init__(self, f: str):
- self.f = f
- self.config = configparser.ConfigParser()
- config = self.config
- config.optionxform=str
- if not os.path.isfile(f):
- config['Preferences'] = {
- 'destructive_pull': False,
- 'destructive_push': False
- }
- config['ServerOrder'] = {
- 'ServerDefault': 1
- }
- config['ServerDefault'] = {
- 'host': '',
- 'uname': 'filesync',
- 'passwd': '',
- 'rootdir': 'filesync',
- 'port': '22',
- }
- config['LocalDevice'] = {
- 'devicename': socket.gethostname()
- }
- config['Sync'] = {}
- with open(f, 'w+') as ff:
- config.write(ff)
- else:
- config.read(f)
- ##
- # The SyncObject represents a file or folder, which should be present on both local and remote machine.
- # This is the backbone for syncing files.
- class SyncObject(object):
- def __init__(self, config: FilesyncConf, rn=None, ln=None):
- self.config = config
- self.rnode = rn
- self.lnode = ln
- def __repr__(self):
- return 'SyncObject(rnode={}, lnode={})'.format(self.get_remote(), self.get_local())
- def __eq__(self, other):
- return self.remote_folder == other.remote_folder and self.local_folder == other.local_folder
- ##
- # Construct the
- #
- def from_remote(self, path: str):
- self.rnode = pathlib.PurePosixPath(path)
- r_pts = self.rnode.parts
- r_pts = [x for x in r_pts if x not in ('.', '..')]
- self.rnode = pathlib.PurePosixPath(*r_pts)
- for key in self.config.config['Sync']:
- try:
- val = self.config.config['Sync'][key]
- except KeyError as e:
- raise
- s_pts = pathlib.Path(key).parts
- s_pts = ['/' if x == '\\' else x for x in s_pts]
- if r_pts[:len(s_pts)] == s_pts:
- self.lnode = pathlib.Path(val).joinpath(*r_pts[len(s_pts):])
- return self
- def from_local(self, path: str):
- self.lnode = pathlib.Path(path)
- l_pts = self.lnode.parts
- l_pts = [x for x in l_pts if x not in ('.', '..')]
- self.lnode = pathlib.Path(*l_pts)
- for key in self.config.config['Sync']:
- try:
- val = self.config.config['Sync'][key]
- except KeyError as e:
- raise
- s_pts = list(pathlib.Path(val).parts)
- s_pts = ['/' if x == '\\' else x for x in s_pts]
- if l_pts[:len(s_pts)] == s_pts:
- self.rnode = pathlib.PurePosixPath(key).joinpath(*l_pts[len(s_pts):])
- return self
- def joinpath(self, *pts):
- return SyncObject(self.config, self.rnode.joinpath(*pts), self.lnode.joinpath(*pts))
- def get_remote(self):
- return self.rnode
- def get_remote_str(self):
- return self.get_remote().as_posix()
- def get_local(self):
- return self.lnode
- def get_local_str(self):
- return self.get_local().as_posix()
- class FileSync(object):
- configdir='./'
- localconf='filesync.conf'
- def __init__(self, config: FilesyncConf):
- self.config = config
- self.server = None
- self.sftp = None
- config = config.config
- server_list = []
- for key in config['ServerOrder']:
- val = config['ServerOrder'][key]
- server_list += [(key, int(val))]
- server_list.sort(key=lambda tup: tup[1])
- server_list.reverse()
- for server, priority in server_list:
- if priority < 0:
- continue
- try:
- self.sftp = pysftp.Connection(config[server]['host'],
- username=config[server]['uname'],
- password=config[server]['passwd'],
- port=int(config[server]['port']))
- except paramiko.ssh_exception.AuthenticationException as e:
- print('Error: couldn\'t connect to server {} (hostname: {})'
- .format(server, config[server]['host']))
- continue
- self.server = server
- if self.sftp is None:
- raise Exception('failed to connect to a server')
- print('using server {}'.format(self.server))
- def init_ftp_storage(self):
- config = self.config.config
- self.sftp.chdir(config[self.server]['rootdir'])
- def get_modtime(self, res: SyncObject):
- config = self.config.config
- try:
- rstats = self.sftp.sftp_client.stat(res.get_remote_str())
- remote_mtime = rstats.st_mtime
- except FileNotFoundError as e:
- rstats = None
- remote_mtime = 0
- lstats = os.stat(res.get_local_str())
- try:
- local_mtime = lstats.st_mtime
- except FileNotFoundError as e:
- lstats = None
- local_mtime = 0
- return int(remote_mtime), int(local_mtime)
- def hard_push_file(self, res: SyncObject):
- self.sftp.put(localpath=res.get_local_str(), remotepath=res.get_remote_str(), callback=None, confirm=True, preserve_mtime=True)
- def hard_pull_file(self, res: SyncObject):
- self.sftp.get(remotepath=res.get_remote_str(), localpath=res.get_local_str(), callback=None, preserve_mtime=True)
- def hard_delete_file(self, res: SyncObject, local=False, remote=False):
- if local:
- os.remove(res.get_local_str())
- if remote:
- self.sftp.sftp_client.remove(res.get_remote_str())
- def hard_delete_dir(self, res: SyncObject, local=False, remote=False):
- if local:
- os.rmdir(res.get_local_str())
- if remote:
- self.sftp.sftp_client.rmdir(res.get_remote_str())
- def push_file(self, res: SyncObject):
- rt, lt = self.get_modtime(res)
- if lt <= rt:
- print('aborting writing of file {}, destination newer than source'.format(res))
- return
- print('writing file: {}'.format(res))
- self.hard_push_file(res)
- def pull_file(self, res: SyncObject):
- rt, lt = self.get_modtime(res)
- if rt <= lt:
- print('aborting writing of file {}, destination newer than source'.format(res))
- return
- print('writing file: {}'.format(res))
- self.hard_pull_file(res)
- def delete_dir(self, res: SyncObject, local=False, remote=False):
- if local:
- print('>>{}'.format(res))
- if os.path.isdir(res.get_local_str()):
- print('entering dir: {}'.format(res))
- for d in os.listdir(res.get_local_str()):
- self.delete_dir(res.joinpath(d), local=local, remote=remote)
- print('deleting directory: {}'.format(res))
- self.hard_delete_dir(res, local=local, remote=remote)
- else:
- print('deleting file: {}'.format(res))
- self.hard_delete_file(res, local=local, remote=remote)
- if remote:
- if self.sftp.isdir(res.get_remote_str()):
- print('entering dir: {}'.format(res))
- for d in self.sftp.listdir(res.get_remote_str()):
- self.delete_dir(res.joinpath(d), local=local, remote=remote)
- print('deleting directory: {}'.format(res))
- self.hard_delete_dir(res, local=local, remote=remote)
- else:
- print('deleting file: {}'.format(res))
- self.hard_delete_file(res, local=local, remote=remote)
- def push_dir(self, res: SyncObject, destructive=False):
- if os.path.isdir(res.get_local_str()):
- print('entering dir: {}'.format(res))
- if not self.sftp.isdir(res.get_remote_str()):
- print('creating dir: {}'.format(res))
- self.sftp.makedirs(res.get_remote_str())
- ldirs = os.listdir(res.get_local_str())
- rdirs = self.sftp.listdir(res.get_remote_str())
- isect = [x for x in ldirs if x in rdirs]
- nisct = [x for x in rdirs if x not in ldirs]
- for d in ldirs:
- self.push_dir(res.joinpath(d), destructive=destructive)
- for d in nisct:
- self.delete_dir(res.joinpath(d), local=False, remote=destructive)
- else:
- print('pushing file: {}'.format(res))
- self.push_file(res)
- def pull_dir(self, res: SyncObject, destructive=False):
- if self.sftp.isdir(res.get_remote_str()):
- print('entering dir: {}'.format(res))
- if not os.path.isdir(res.get_local_str()):
- print('creating dir: {}'.format(res))
- os.path.makedirs(res.get_local_str())
- ldirs = os.listdir(res.get_local_str())
- rdirs = self.sftp.listdir(res.get_remote_str())
- isect = [x for x in rdirs if x in ldirs]
- nisct = [x for x in ldirs if x not in rdirs]
- for d in rdirs:
- self.pull_dir(res.joinpath(d), destructive=destructive)
- for d in nisct:
- self.delete_dir(res.joinpath(d), local=destructive, remote=False)
- else:
- print('pulling file: {}'.format(res))
- self.pull_file(res)
- def push(self, od=None):
- config = self.config.config
- d = bool(config['Preferences']['destructive_push'])
- if od != None:
- d = od
- for key in config['Sync']:
- self.push_dir(SyncObject(self.config).from_remote(key), destructive=d)
- def pull(self, od=None):
- config = self.config.config
- d = bool(config['Preferences']['destructive_pull'])
- if od != None:
- d = od
- for key in config['Sync']:
- self.pull_dir(SyncObject(self.config).from_remote(key), destructive=d)
- def main(argc, argv):
- destructive = False
- verbose = 0
- config = 'filesync.conf'
- opts, args = getopt.getopt(argv[1:], "hvdc:", ["help", "verbose", "destructive", "config="])
- for o, a in opts:
- if o in ("-h", "--help"):
- print("HELP!")
- return
- elif o in ("-v", "--verbose"):
- verbose += 1
- elif o in ("-d", "--destructive"):
- destructive = True
- elif o in ("-c", "--config"):
- config = a
- if len(args) != 1:
- return
- fs = FileSync(FilesyncConf(config))
- if args[0] == "push":
- fs.push(od=destructive)
- elif args[0] == "pull":
- fs.pull(od=destructive)
- if __name__ == '__main__':
- main(len(sys.argv), sys.argv);
Add Comment
Please, Sign In to add comment