Guest User

Untitled

a guest
Oct 22nd, 2018
109
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 11.30 KB | None | 0 0
  1. #!/usr/bin/env python3
  2.  
  3. ##
  4. # @file filesync.py
  5. # @author redxef
  6. # @version 0.0.0-r0
  7. # @since 2018-10-22
  8. #
  9. # @brief A simple sync utility for sftp servers.
  10. # This utility is meant to be run in the background and called periodically and
  11. # synchronizes folders on an sftp server with the local folders specified in the config.
  12. # Multiple folders (or files) can be selected and will then be synced with the server.
  13. # This utility is only really useful with multiple devices for syncing local folders.
  14. # Prime examples would be the Documents folder.
  15. ##
  16.  
  17. import sys
  18. import os
  19. import socket
  20. import pathlib
  21. import configparser
  22. import pysftp
  23. import getopt
  24. import paramiko
  25.  
  26.  
  27. ##
  28. # Stores the configuration and creates a default configuration file in case there is none.
  29. class FilesyncConf(object):
  30.  
  31. def __init__(self, f: str):
  32. self.f = f
  33. self.config = configparser.ConfigParser()
  34. config = self.config
  35. config.optionxform=str
  36. if not os.path.isfile(f):
  37. config['Preferences'] = {
  38. 'destructive_pull': False,
  39. 'destructive_push': False
  40. }
  41. config['ServerOrder'] = {
  42. 'ServerDefault': 1
  43. }
  44. config['ServerDefault'] = {
  45. 'host': '',
  46. 'uname': 'filesync',
  47. 'passwd': '',
  48. 'rootdir': 'filesync',
  49. 'port': '22',
  50. }
  51. config['LocalDevice'] = {
  52. 'devicename': socket.gethostname()
  53. }
  54. config['Sync'] = {}
  55.  
  56. with open(f, 'w+') as ff:
  57. config.write(ff)
  58. else:
  59. config.read(f)
  60.  
  61.  
  62. ##
  63. # The SyncObject represents a file or folder, which should be present on both local and remote machine.
  64. # This is the backbone for syncing files.
  65. class SyncObject(object):
  66.  
  67. def __init__(self, config: FilesyncConf, rn=None, ln=None):
  68. self.config = config
  69. self.rnode = rn
  70. self.lnode = ln
  71.  
  72. def __repr__(self):
  73. return 'SyncObject(rnode={}, lnode={})'.format(self.get_remote(), self.get_local())
  74.  
  75. def __eq__(self, other):
  76. return self.remote_folder == other.remote_folder and self.local_folder == other.local_folder
  77.  
  78. ##
  79. # Construct the
  80. #
  81. def from_remote(self, path: str):
  82. self.rnode = pathlib.PurePosixPath(path)
  83. r_pts = self.rnode.parts
  84. r_pts = [x for x in r_pts if x not in ('.', '..')]
  85. self.rnode = pathlib.PurePosixPath(*r_pts)
  86.  
  87. for key in self.config.config['Sync']:
  88. try:
  89. val = self.config.config['Sync'][key]
  90. except KeyError as e:
  91. raise
  92. s_pts = pathlib.Path(key).parts
  93. s_pts = ['/' if x == '\\' else x for x in s_pts]
  94.  
  95. if r_pts[:len(s_pts)] == s_pts:
  96. self.lnode = pathlib.Path(val).joinpath(*r_pts[len(s_pts):])
  97. return self
  98.  
  99. def from_local(self, path: str):
  100. self.lnode = pathlib.Path(path)
  101. l_pts = self.lnode.parts
  102. l_pts = [x for x in l_pts if x not in ('.', '..')]
  103. self.lnode = pathlib.Path(*l_pts)
  104.  
  105. for key in self.config.config['Sync']:
  106. try:
  107. val = self.config.config['Sync'][key]
  108. except KeyError as e:
  109. raise
  110. s_pts = list(pathlib.Path(val).parts)
  111. s_pts = ['/' if x == '\\' else x for x in s_pts]
  112.  
  113. if l_pts[:len(s_pts)] == s_pts:
  114. self.rnode = pathlib.PurePosixPath(key).joinpath(*l_pts[len(s_pts):])
  115. return self
  116.  
  117. def joinpath(self, *pts):
  118. return SyncObject(self.config, self.rnode.joinpath(*pts), self.lnode.joinpath(*pts))
  119.  
  120. def get_remote(self):
  121. return self.rnode
  122.  
  123. def get_remote_str(self):
  124. return self.get_remote().as_posix()
  125.  
  126. def get_local(self):
  127. return self.lnode
  128.  
  129. def get_local_str(self):
  130. return self.get_local().as_posix()
  131.  
  132.  
  133. class FileSync(object):
  134.  
  135. configdir='./'
  136. localconf='filesync.conf'
  137.  
  138. def __init__(self, config: FilesyncConf):
  139. self.config = config
  140. self.server = None
  141. self.sftp = None
  142.  
  143. config = config.config
  144.  
  145. server_list = []
  146. for key in config['ServerOrder']:
  147. val = config['ServerOrder'][key]
  148. server_list += [(key, int(val))]
  149. server_list.sort(key=lambda tup: tup[1])
  150. server_list.reverse()
  151.  
  152. for server, priority in server_list:
  153. if priority < 0:
  154. continue
  155. try:
  156. self.sftp = pysftp.Connection(config[server]['host'],
  157. username=config[server]['uname'],
  158. password=config[server]['passwd'],
  159. port=int(config[server]['port']))
  160. except paramiko.ssh_exception.AuthenticationException as e:
  161. print('Error: couldn\'t connect to server {} (hostname: {})'
  162. .format(server, config[server]['host']))
  163. continue
  164. self.server = server
  165. if self.sftp is None:
  166. raise Exception('failed to connect to a server')
  167. print('using server {}'.format(self.server))
  168.  
  169. def init_ftp_storage(self):
  170. config = self.config.config
  171. self.sftp.chdir(config[self.server]['rootdir'])
  172.  
  173. def get_modtime(self, res: SyncObject):
  174. config = self.config.config
  175. try:
  176. rstats = self.sftp.sftp_client.stat(res.get_remote_str())
  177. remote_mtime = rstats.st_mtime
  178. except FileNotFoundError as e:
  179. rstats = None
  180. remote_mtime = 0
  181. lstats = os.stat(res.get_local_str())
  182. try:
  183. local_mtime = lstats.st_mtime
  184. except FileNotFoundError as e:
  185. lstats = None
  186. local_mtime = 0
  187. return int(remote_mtime), int(local_mtime)
  188.  
  189. def hard_push_file(self, res: SyncObject):
  190. self.sftp.put(localpath=res.get_local_str(), remotepath=res.get_remote_str(), callback=None, confirm=True, preserve_mtime=True)
  191.  
  192. def hard_pull_file(self, res: SyncObject):
  193. self.sftp.get(remotepath=res.get_remote_str(), localpath=res.get_local_str(), callback=None, preserve_mtime=True)
  194.  
  195. def hard_delete_file(self, res: SyncObject, local=False, remote=False):
  196. if local:
  197. os.remove(res.get_local_str())
  198. if remote:
  199. self.sftp.sftp_client.remove(res.get_remote_str())
  200.  
  201. def hard_delete_dir(self, res: SyncObject, local=False, remote=False):
  202. if local:
  203. os.rmdir(res.get_local_str())
  204. if remote:
  205. self.sftp.sftp_client.rmdir(res.get_remote_str())
  206.  
  207. def push_file(self, res: SyncObject):
  208. rt, lt = self.get_modtime(res)
  209. if lt <= rt:
  210. print('aborting writing of file {}, destination newer than source'.format(res))
  211. return
  212. print('writing file: {}'.format(res))
  213. self.hard_push_file(res)
  214.  
  215. def pull_file(self, res: SyncObject):
  216. rt, lt = self.get_modtime(res)
  217. if rt <= lt:
  218. print('aborting writing of file {}, destination newer than source'.format(res))
  219. return
  220. print('writing file: {}'.format(res))
  221. self.hard_pull_file(res)
  222.  
  223. def delete_dir(self, res: SyncObject, local=False, remote=False):
  224. if local:
  225. print('>>{}'.format(res))
  226. if os.path.isdir(res.get_local_str()):
  227. print('entering dir: {}'.format(res))
  228. for d in os.listdir(res.get_local_str()):
  229. self.delete_dir(res.joinpath(d), local=local, remote=remote)
  230. print('deleting directory: {}'.format(res))
  231. self.hard_delete_dir(res, local=local, remote=remote)
  232. else:
  233. print('deleting file: {}'.format(res))
  234. self.hard_delete_file(res, local=local, remote=remote)
  235.  
  236. if remote:
  237. if self.sftp.isdir(res.get_remote_str()):
  238. print('entering dir: {}'.format(res))
  239. for d in self.sftp.listdir(res.get_remote_str()):
  240. self.delete_dir(res.joinpath(d), local=local, remote=remote)
  241. print('deleting directory: {}'.format(res))
  242. self.hard_delete_dir(res, local=local, remote=remote)
  243. else:
  244. print('deleting file: {}'.format(res))
  245. self.hard_delete_file(res, local=local, remote=remote)
  246.  
  247. def push_dir(self, res: SyncObject, destructive=False):
  248. if os.path.isdir(res.get_local_str()):
  249. print('entering dir: {}'.format(res))
  250. if not self.sftp.isdir(res.get_remote_str()):
  251. print('creating dir: {}'.format(res))
  252. self.sftp.makedirs(res.get_remote_str())
  253. ldirs = os.listdir(res.get_local_str())
  254. rdirs = self.sftp.listdir(res.get_remote_str())
  255. isect = [x for x in ldirs if x in rdirs]
  256. nisct = [x for x in rdirs if x not in ldirs]
  257. for d in ldirs:
  258. self.push_dir(res.joinpath(d), destructive=destructive)
  259. for d in nisct:
  260. self.delete_dir(res.joinpath(d), local=False, remote=destructive)
  261. else:
  262. print('pushing file: {}'.format(res))
  263. self.push_file(res)
  264.  
  265. def pull_dir(self, res: SyncObject, destructive=False):
  266. if self.sftp.isdir(res.get_remote_str()):
  267. print('entering dir: {}'.format(res))
  268. if not os.path.isdir(res.get_local_str()):
  269. print('creating dir: {}'.format(res))
  270. os.path.makedirs(res.get_local_str())
  271. ldirs = os.listdir(res.get_local_str())
  272. rdirs = self.sftp.listdir(res.get_remote_str())
  273. isect = [x for x in rdirs if x in ldirs]
  274. nisct = [x for x in ldirs if x not in rdirs]
  275. for d in rdirs:
  276. self.pull_dir(res.joinpath(d), destructive=destructive)
  277. for d in nisct:
  278. self.delete_dir(res.joinpath(d), local=destructive, remote=False)
  279. else:
  280. print('pulling file: {}'.format(res))
  281. self.pull_file(res)
  282.  
  283. def push(self, od=None):
  284. config = self.config.config
  285. d = bool(config['Preferences']['destructive_push'])
  286. if od != None:
  287. d = od
  288. for key in config['Sync']:
  289. self.push_dir(SyncObject(self.config).from_remote(key), destructive=d)
  290.  
  291. def pull(self, od=None):
  292. config = self.config.config
  293. d = bool(config['Preferences']['destructive_pull'])
  294. if od != None:
  295. d = od
  296. for key in config['Sync']:
  297. self.pull_dir(SyncObject(self.config).from_remote(key), destructive=d)
  298.  
  299. def main(argc, argv):
  300. destructive = False
  301. verbose = 0
  302. config = 'filesync.conf'
  303. opts, args = getopt.getopt(argv[1:], "hvdc:", ["help", "verbose", "destructive", "config="])
  304.  
  305. for o, a in opts:
  306. if o in ("-h", "--help"):
  307. print("HELP!")
  308. return
  309. elif o in ("-v", "--verbose"):
  310. verbose += 1
  311. elif o in ("-d", "--destructive"):
  312. destructive = True
  313. elif o in ("-c", "--config"):
  314. config = a
  315.  
  316. if len(args) != 1:
  317. return
  318.  
  319. fs = FileSync(FilesyncConf(config))
  320.  
  321. if args[0] == "push":
  322. fs.push(od=destructive)
  323. elif args[0] == "pull":
  324. fs.pull(od=destructive)
  325.  
  326. if __name__ == '__main__':
  327. main(len(sys.argv), sys.argv);
Add Comment
Please, Sign In to add comment