Advertisement
Guest User

Untitled

a guest
Mar 17th, 2018
283
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 9.26 KB | None | 0 0
  1. from instance_factory import collect_builders
  2. from instance_factory import collect_bug_trackers
  3. import pymongo
  4. from tools.log_parser import LogParser
  5. import json
  6.  
  7.  
  8. class MongoInstance():
  9. def __init__(self):
  10. self.client = pymongo.MongoClient(
  11. r"mongodb://heroku_6m3l526r:7mesdarsb1n8jnfvv5lv913j4r@ds259085.mlab.com:59085/heroku_6m3l526r")
  12. pymongo.db = self.client.heroku_6m3l526r
  13.  
  14. with open('Config_for_database_repositories.json', 'r') as file:
  15. Repositories = file.read()
  16. parsed__Repository = json.loads(Repositories)
  17. self.List_Travis_Rep = parsed__Repository.get("Travis_Repository")
  18. self.List_Github_Rep = parsed__Repository.get("Github_Repository")
  19.  
  20. DB_Builders = collect_builders()
  21. #DB_Bugs = collect_bug_trackers()
  22. self.travis = DB_Builders[1]
  23. #self.jenkins = DB_Builders[0]
  24. #self.github = DB_Bugs[0]
  25. #self.jira = DB_Bugs[1]
  26.  
  27. def show_docs(self):
  28. for docs in pymongo.db.travis.find():
  29. print(docs)
  30. print(docs['Repository'])
  31.  
  32. def insert_all_from_jenkins(self):
  33. builds = self.jenkins.get_all_logs("numpy")
  34. collection = pymongo.db["jenkins"]
  35. server_name = self.jenkins.server_name
  36.  
  37. for build in builds:
  38. raw = {
  39. "log": build.get_console(),
  40. "time": str(build.get_duration()),
  41. "status": build.get_status(),
  42. "author": build.get_causes()[0]["shortDescription"],
  43. "revision": build.get_revision(),
  44. "server": server_name,
  45. "name": build.baseurl
  46. }
  47. if not collection.find_one(raw):
  48. collection.insert(raw)
  49.  
  50. def insert_last_from_travis(self):
  51. for rep in range(len(self.List_Travis_Rep)):
  52. print('Working with...', self.List_Travis_Rep[rep])
  53. log = self.travis.get_all_fails(self.List_Travis_Rep[rep])
  54. for key in log:
  55. print("%s -> %s" % (key, log[key]))
  56.  
  57. '''
  58. Log_id = log["id"]
  59. Build_lanquage = log["language"]
  60. Job_id = log["job_id"]
  61. Committer_name = log["committer_name"]
  62. Committer_email = log["committer_email"]
  63. Commit_message = log["message"]
  64. Committed_time = log["committed_at"]
  65. Parsed_log = LogParser().parse_travis_log(log["content"].rstrip())
  66. if (Log_id == -1):
  67. print('Connection problem with travis')
  68. continue
  69. doc = pymongo.db.travis.find_one(
  70. {
  71. 'Log_id': Log_id
  72. }
  73. )
  74.  
  75. if (doc):
  76. print("Log already put in (", Log_id, ')')
  77. else:
  78. print("Inserting log... (", Log_id, ')')
  79. pymongo.db.travis.insert(
  80. {
  81.  
  82. 'Repository': self.List_Travis_Rep[rep],
  83. 'Build_language': Build_lanquage,
  84. 'Job_id': Job_id,
  85. 'Log_committer_name': Committer_name,
  86. 'Message': Commit_message,
  87. 'Log_committer_email': Committer_email,
  88. 'Log_committed_time': Committed_time,
  89. 'Log_id': Log_id,
  90. 'Log': Parsed_log
  91.  
  92. }
  93. )
  94. '''
  95. def insert_all_from_travis(self):
  96. for rep in range(len(self.List_Travis_Rep)):
  97. print('Working with...', self.List_Travis_Rep[rep])
  98.  
  99. for log in self.travis.get_all_fails(self.List_Travis_Rep[rep]):
  100. Log_id = log["id"]
  101. Build_lanquage = log["language"]
  102. Job_id = log["job_id"]
  103. Committer_name = log["committer_name"]
  104. Committer_email = log["committer_email"]
  105. Commit_message = log["message"]
  106. Committed_time = log["committed_at"]
  107. Parsed_log = LogParser().parse_travis_log(log["content"].rstrip())
  108. if (Log_id == -1):
  109. print('Connection problem with travis')
  110. continue
  111. doc = pymongo.db.travis.find_one(
  112. {
  113.  
  114. 'Log_id': Log_id
  115.  
  116. }
  117. )
  118. if (doc):
  119. print("Log already put in (", Log_id, ')')
  120. continue
  121. else:
  122. print("Inserting log... (", Log_id, ')')
  123. pymongo.db.travis.insert(
  124. {
  125.  
  126. 'Repository': self.List_Travis_Rep[rep],
  127. 'Build_language': Build_lanquage,
  128. 'Job_id': Job_id,
  129. 'Log_committer_name': Committer_name,
  130. 'Message': Commit_message,
  131. 'Log_committer_email': Committer_email,
  132. 'Log_committed_time': Committed_time,
  133. 'Log_id': Log_id,
  134. 'Log': Parsed_log
  135.  
  136. }
  137. )
  138.  
  139. def insert_all_from_jira(self):
  140. """
  141. Inserts all issues from jira with credentials given in instances.json.
  142. Structure of issue document can be find in jira_tracker.py.
  143. """
  144. issues = self.jira.get_all_issues()
  145. collection = pymongo.db.jira
  146.  
  147. for issue in issues:
  148. if not collection.find_one(issue):
  149. collection.insert(issue)
  150.  
  151. def insert_all_from_github(self):
  152. for rep in range(len(self.List_Github_Rep)):
  153. print('Working with...', self.List_Github_Rep[rep])
  154.  
  155. issues = self.github.get_all_issues(self.List_Github_Rep[rep])
  156. for issue in issues:
  157. closed_by_login = ''
  158. closed_by_id = ''
  159. repository_url = issue['repository_url']
  160. id_issue = issue['id']
  161. number = issue['number']
  162. title = issue['title'].strip()
  163. user_login = issue['user']['login']
  164. user_id = issue['user']['id']
  165. created_at = issue['created_at']
  166. updated_at = issue['updated_at']
  167. closed_at = issue['closed_at']
  168. if (issue['closed_by']):
  169. closed_by_login = issue['closed_by']['login']
  170. closed_by_id = issue['closed_by']['id']
  171. body = issue['body'].strip()
  172. doc = pymongo.db.github.find_one(
  173. {
  174.  
  175. 'Issue_id': id_issue
  176.  
  177. }
  178. )
  179. if (doc):
  180. print("Issue already put in (", id_issue, ')')
  181. continue
  182. else:
  183. print('Inserting issue ...(', id_issue, ')')
  184. pymongo.db.github.insert(
  185. {
  186. 'Repository': self.List_Github_Rep[rep],
  187. 'Repository_url': repository_url,
  188. 'Issue_id': id_issue,
  189. 'Issue_number': number,
  190. 'Issue_title': title,
  191. 'User_login': user_login,
  192. 'User_id': user_id,
  193. 'Created_at': created_at,
  194. 'Updated_at': updated_at,
  195. 'Closed_at': closed_at,
  196. 'Closed_by_login': closed_by_login,
  197. 'Closed_by_id': closed_by_id,
  198. 'Body': body,
  199.  
  200. }
  201. )
  202.  
  203.  
  204. def update_jenkins(self, id, **kwargs):
  205. pymongo.db.jenkins.find_one_and_update({'_id': id}, update={'$set': kwargs})
  206.  
  207. def update_jira(self, id, **kwargs):
  208. pymongo.db.jira.find_one_and_update({'_id': id}, update={'$set': kwargs})
  209.  
  210. def update_travis(self, id, ** kwargs):
  211. pymongo.db.travis.find_one_and_update({'_id': id}, update={'$set': kwargs})
  212.  
  213. def update_github(self, id, **kwargs):
  214. pymongo.db.github.find_one_and_update({'_id': id}, update={'$set': kwargs})
  215.  
  216. def get_travis_logs(self):
  217. pass
  218.  
  219. def get_artifacts_from_db(self, instance_name):
  220. collection = pymongo.db[instance_name]
  221. docs = collection.find({})
  222. return list(map(lambda x: x, docs))
  223.  
  224. def get_all_collection(self):
  225. data = dict.fromkeys(["travis", "jenkins", "github", "jira"])
  226. for instance in data.keys():
  227. data[instance] = self.get_artifacts_from_db(instance)
  228. return data
  229.  
  230. def close(self):
  231. self.client.close()
  232.  
  233. if __name__ == "__main__":
  234. DbObj = MongoInstance()
  235. DbObj.insert_last_from_travis()
  236. # DbObj.close()
  237. # DbObj.insert_all_from_jenkins()
  238. # pymongo.db.create_collection("travis")
  239. # pymongo.db.create_collection("github")
  240. # insert_all_from_travis()
  241. #DbObj.insert_last_from_jenkins()
  242. # insert_all_from_github()
  243. # show_docs()
  244. # update_github()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement