Advertisement
Guest User

Untitled

a guest
Feb 1st, 2017
159
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 8.17 KB | None | 0 0
  1. #!/usr/bin/python
  2. # -*- coding: utf-8 -*-
  3.  
  4. # (c) 2017, Valeriy Solovyov <weldpua2008@gmail.com>
  5.  
  6. DOCUMENTATION = '''
  7. ---
  8. module: gridfs_artifact
  9. short_description: Downloads files from MongoDB GridFS to node
  10. description:
  11. - Downloads files from GridFS. The remote
  12. server I(must) have direct access to the remote resource.
  13. options:
  14. database:
  15. required: True
  16. description:
  17. - the name of a getent database supported by the target system (passwd, group,
  18. hosts, etc).
  19. key:
  20. required: False
  21. default: ''
  22. description:
  23. - key from which to return values from the specified database, otherwise the
  24. full contents are returned.
  25. split:
  26. required: False
  27. default: None
  28. description:
  29. - "character used to split the database values into lists/arrays such as ':' or '\t', otherwise it will try to pick one depending on the database"
  30. fail_key:
  31. required: False
  32. default: True
  33. description:
  34. - If a supplied key is missing this will make the task fail if True
  35. notes:
  36. - "Not all databases support enumeration, check system documentation for details"
  37. requirements: [ pymongo ]
  38. author: "Valeriy Solovyov"
  39. '''
  40.  
  41. EXAMPLES = '''
  42. # get file
  43. - gridfs_artifact: url="test.txt" dst="/tmp/test2.txt" database="cdn_storage" login_host="login.host" force=yes
  44. '''
  45.  
  46. try:
  47. from pymongo.errors import ConnectionFailure
  48. from pymongo.errors import OperationFailure
  49. from pymongo import version as PyMongoVersion
  50. from pymongo import MongoClient
  51. import gridfs
  52. except ImportError:
  53. try: # for older PyMongo 2.2
  54. from pymongo import Connection as MongoClient
  55. import gridfs
  56. except ImportError:
  57. pymongo_found = False
  58. else:
  59. pymongo_found = True
  60. else:
  61. pymongo_found = True
  62.  
  63.  
  64. # from distutils.version import LooseVersion
  65. # import datetime
  66. import os
  67. import hashlib
  68.  
  69. __author__ = 'weldpua2008@gmail'
  70.  
  71. # ==============================================================
  72. # main
  73.  
  74. class MD5Error(Exception):
  75. pass
  76.  
  77.  
  78. class MD5CompareError(Exception):
  79. pass
  80.  
  81.  
  82.  
  83. def connect_to_db(module):
  84. replica_set = module.params['replica_set']
  85. ssl = module.params['ssl']
  86. db_name = module.params['database']
  87. # read_preferences = module.params['read_preferences']
  88. login_host = module.params["login_host"]
  89. login_password = module.params["login_password"]
  90. login_user = module.params["login_user"]
  91. port = module.params['port']
  92. replica_set_hosts = module.params['replica_set_hosts']
  93.  
  94. try:
  95. if replica_set:
  96. if not isinstance(replica_set_hosts, list):
  97. module.fail_json(msg="replica_set_hosts %s wasn't list" % str(replica_set_hosts))
  98. if len(replica_set_hosts) < 1:
  99. module.fail_json(msg="replica_set_hosts is empty" % str(replica_set_hosts))
  100. _replica_set_hosts = ','.join(replica_set_hosts)
  101.  
  102. client = MongoClient('mongodb://%s/' % _replica_set_hosts, int(port), replicaset=replica_set, ssl=ssl)
  103. else:
  104. if login_host is None:
  105. module.fail_json(msg="login_host is empty" % str(replica_set_hosts))
  106. client = MongoClient(login_host, int(port), ssl=ssl)
  107.  
  108. if login_user is not None and login_password is not None:
  109. client.admin.authenticate(login_user, login_password)
  110.  
  111. except ConnectionFailure, error:
  112. module.fail_json(msg="unable to connect to database: %s" % str(error))
  113.  
  114. # db = MongoClient('mongodb://%s/?replicaSet=%s' % (replica_set_hosts, replica_set),read_preference=pymongo.ReadPreference.NEAREST).rdecdn_storage
  115. db = client[db_name]
  116. fs = gridfs.GridFS(db)
  117. return fs
  118.  
  119. def check_md5(store_path, md5):
  120. if hashlib.md5(open(store_path, 'rb').read()).hexdigest() != md5:
  121. raise MD5CompareError("md5 is not equal")
  122.  
  123. def get_file(fs, key, store_path, force=False, md5=None, check_md5sum=False):
  124.  
  125. _md5 = fs.get_version(key).md5
  126. # _md5 = None
  127. # try:
  128. # _md5 = fs.get_version(key).md5
  129. # except Exception as error:
  130. # pass
  131.  
  132. if not os.path.exists(os.path.dirname(store_path)):
  133. os.makedirs(os.path.dirname(store_path))
  134. if not os.path.exists(store_path) or force:
  135. with open(store_path, "wb") as f:
  136. f.write(fs.get_last_version(key).read())
  137. try:
  138. if md5 is not None:
  139. check_md5(store_path=store_path, md5=md5)
  140. elif check_md5sum:
  141. check_md5(store_path=store_path, md5=_md5)
  142.  
  143. except MD5CompareError as error:
  144. os.remove(store_path)
  145. raise MD5CompareError(str(error))
  146. except Exception as error:
  147. os.remove(store_path)
  148. raise MD5Error("Can't calculate md5 %s" % error)
  149. return True, _md5
  150. return False, _md5
  151.  
  152.  
  153.  
  154. def save_file(fs, key, source_path, clear_versions=False):
  155. md5 = None
  156. if os.path.exists(source_path):
  157. if clear_versions:
  158. try:
  159. while fs.exists(filename=key):
  160. fp = fs.get_last_version(key)
  161. fs.delete(fp._id)
  162. except Exception:
  163. pass
  164. with fs.new_file(filename=key) as fp:
  165. with open(source_path, "rb") as data:
  166. fp.write(data)
  167.  
  168. return True, md5
  169. else:
  170. raise ValueError("Not exists '%s'" % source_path)
  171.  
  172.  
  173.  
  174. def main():
  175.  
  176. module = AnsibleModule(
  177. argument_spec=dict(
  178. url=dict(required=True, aliases=['key', 'file']),
  179. database=dict(required=True, aliases=['db']),
  180. store_path=dict(default=None, required=False, aliases=['dest','dst']),
  181. source_path=dict(default=None, required=False, aliases=['src']),
  182. force=dict(default=False, required=False),
  183. md5sum=dict(default=None, required=False, aliases=['md5']),
  184. check_md5sum=dict(default=True, required=False, aliases=['check_md5sum']),
  185. clear_versions=dict(default=True, required=False),
  186. port=dict(default="27017", required=False),
  187. replica_set=dict(required=False),
  188. replica_set_hosts=dict(type='list', default=[], required=False),
  189. login_host=dict(default=None, required=False),
  190. login_user=dict(default=None, required=False),
  191. login_password=dict(default=None, required=False),
  192. ssl=dict(default=False, required=False),
  193. # state=dict(default="present", choices=["absent", "present"]),
  194. ),
  195. supports_check_mode=True
  196. )
  197. if not pymongo_found:
  198. module.fail_json(msg="the python pymongo module is required")
  199.  
  200. url = module.params['url'].strip()
  201. store_path = module.params['store_path']
  202. source_path = module.params['source_path']
  203. force = module.params['force']
  204. clear_versions = module.params['clear_versions']
  205. md5sum = module.params['md5sum']
  206. check_md5sum = module.params['check_md5sum']
  207. kwargs = {"changed": False}
  208. try:
  209. fs = connect_to_db(module=module)
  210. if store_path is not None:
  211. changed, md5 = get_file(fs=fs, key=url, store_path=store_path.strip(), force=force, md5=md5sum, check_md5sum=check_md5sum)
  212. kwargs = {"url":url, "store_path": store_path, "src": url, "changed": changed, "md5": md5, "ok": True}
  213. elif source_path is not None:
  214. changed, md5 = save_file(fs=fs, key=url, source_path=source_path.strip(), clear_versions=clear_versions)
  215. kwargs = {"url":url, "source_path": source_path, "clear_versions": clear_versions, "changed": changed, "md5": md5, "ok": True}
  216.  
  217. except gridfs.errors.CorruptGridFile as error:
  218. module.fail_json(msg="file %s is malformed %s" % (url, str(error)) )
  219. except gridfs.errors.FileExists as error:
  220. module.fail_json(msg="Trying to create a file %s that already exists: %s" % (url, str(error)) )
  221. except gridfs.errors.NoFile:
  222. module.fail_json(msg="Tying to get a non-existent file '%s'" % url )
  223. except Exception as error:
  224. module.fail_json(msg="There was an error %s" % str(error))
  225. module.exit_json(**kwargs)
  226.  
  227. # import module snippets
  228. from ansible.module_utils.basic import *
  229. # from ansible.module_utils.urls import *
  230. if __name__ == '__main__':
  231. main()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement