Advertisement
Guest User

Untitled

a guest
Mar 18th, 2018
253
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.64 KB | None | 0 0
  1. from instance_factory import collect_builders
  2. from instance_factory import collect_bug_trackers
  3. import pymongo
  4. from tools.log_parser import LogParser
  5. import json
  6.  
  7.  
  8. class MongoInstance():
  9. def __init__(self):
  10. self.client = pymongo.MongoClient(
  11. r"mongodb://heroku_6m3l526r:7mesdarsb1n8jnfvv5lv913j4r@ds259085.mlab.com:59085/heroku_6m3l526r")
  12. pymongo.db = self.client.heroku_6m3l526r
  13.  
  14. with open('Config_for_database_repositories.json', 'r') as file:
  15. Repositories = file.read()
  16. parsed__Repository = json.loads(Repositories)
  17. self.List_Travis_Rep = parsed__Repository.get("Travis_Repository")
  18. self.List_Github_Rep = parsed__Repository.get("Github_Repository")
  19.  
  20. DB_Builders = collect_builders()
  21. #DB_Bugs = collect_bug_trackers()
  22. self.travis = DB_Builders[1]
  23. #self.jenkins = DB_Builders[0]
  24. #self.github = DB_Bugs[0]
  25. #self.jira = DB_Bugs[1]
  26.  
  27. def show_docs(self):
  28. for docs in pymongo.db.travis.find():
  29. print(docs)
  30. print(docs['Repository'])
  31.  
  32. def insert_all_from_jenkins(self):
  33. builds = self.jenkins.get_all_logs("numpy")
  34. collection = pymongo.db["jenkins"]
  35. server_name = self.jenkins.server_name
  36.  
  37. for build in builds:
  38. raw = {
  39. "log": build.get_console(),
  40. "time": str(build.get_duration()),
  41. "status": build.get_status(),
  42. "author": build.get_causes()[0]["shortDescription"],
  43. "revision": build.get_revision(),
  44. "server": server_name,
  45. "name": build.baseurl
  46. }
  47. if not collection.find_one(raw):
  48. collection.insert(raw)
  49.  
  50. def insert_from_travis(self):
  51. for rep in range(len(self.List_Travis_Rep)):
  52. print('Working with...', self.List_Travis_Rep[rep])
  53. logs = self.travis.get_all_fails(self.List_Travis_Rep[rep])
  54. for key in logs:
  55. Lib = logs[key]
  56. Log_id = key
  57. Build_lanquage = Lib['job']['config']['language']
  58. Job_id = Lib['job']['id']
  59. Committer_name = Lib['commit']['author_name']
  60. Committer_email = Lib['commit']['author_email']
  61. Commit_message = Lib['commit']['message']
  62. Committed_time = Lib['commit']['committed_at']
  63. Parsed_log = LogParser().parse_travis_log(Lib['log'].rstrip())
  64.  
  65. if (Log_id == -1):
  66. print('Connection problem with travis')
  67. continue
  68. doc = pymongo.db.travis.find_one(
  69. {
  70.  
  71. 'Log_id': Log_id
  72.  
  73. }
  74. )
  75. if (doc):
  76. print("Log already put in (", Log_id, ')')
  77. continue
  78. else:
  79. print("Inserting log... (", Log_id, ')')
  80. pymongo.db.travis.insert(
  81. {
  82.  
  83. 'Repository': self.List_Travis_Rep[rep],
  84. 'Build_language': Build_lanquage,
  85. 'Job_id': Job_id,
  86. 'Log_committer_name': Committer_name,
  87. 'Message': Commit_message,
  88. 'Log_committer_email': Committer_email,
  89. 'Log_committed_time': Committed_time,
  90. 'Log_id': Log_id,
  91. 'Log': Parsed_log
  92.  
  93. }
  94. )
  95.  
  96.  
  97. def insert_all_from_jira(self):
  98. """
  99. Inserts all issues from jira with credentials given in instances.json.
  100. Structure of issue document can be find in jira_tracker.py.
  101. """
  102. issues = self.jira.get_all_issues()
  103. collection = pymongo.db.jira
  104.  
  105. for issue in issues:
  106. if not collection.find_one(issue):
  107. collection.insert(issue)
  108.  
  109. def insert_all_from_github(self):
  110. for rep in range(len(self.List_Github_Rep)):
  111. print('Working with...', self.List_Github_Rep[rep])
  112.  
  113. issues = self.github.get_all_issues(self.List_Github_Rep[rep])
  114. for issue in issues:
  115. closed_by_login = ''
  116. closed_by_id = ''
  117. repository_url = issue['repository_url']
  118. id_issue = issue['id']
  119. number = issue['number']
  120. title = issue['title'].strip()
  121. user_login = issue['user']['login']
  122. user_id = issue['user']['id']
  123. created_at = issue['created_at']
  124. updated_at = issue['updated_at']
  125. closed_at = issue['closed_at']
  126. if (issue['closed_by']):
  127. closed_by_login = issue['closed_by']['login']
  128. closed_by_id = issue['closed_by']['id']
  129. body = issue['body'].strip()
  130. doc = pymongo.db.github.find_one(
  131. {
  132.  
  133. 'Issue_id': id_issue
  134.  
  135. }
  136. )
  137. if (doc):
  138. print("Issue already put in (", id_issue, ')')
  139. continue
  140. else:
  141. print('Inserting issue ...(', id_issue, ')')
  142. pymongo.db.github.insert(
  143. {
  144. 'Repository': self.List_Github_Rep[rep],
  145. 'Repository_url': repository_url,
  146. 'Issue_id': id_issue,
  147. 'Issue_number': number,
  148. 'Issue_title': title,
  149. 'User_login': user_login,
  150. 'User_id': user_id,
  151. 'Created_at': created_at,
  152. 'Updated_at': updated_at,
  153. 'Closed_at': closed_at,
  154. 'Closed_by_login': closed_by_login,
  155. 'Closed_by_id': closed_by_id,
  156. 'Body': body,
  157.  
  158. }
  159. )
  160.  
  161.  
  162. def update_jenkins(self, id, **kwargs):
  163. pymongo.db.jenkins.find_one_and_update({'_id': id}, update={'$set': kwargs})
  164.  
  165. def update_jira(self, id, **kwargs):
  166. pymongo.db.jira.find_one_and_update({'_id': id}, update={'$set': kwargs})
  167.  
  168. def update_travis(self, id, ** kwargs):
  169. pymongo.db.travis.find_one_and_update({'_id': id}, update={'$set': kwargs})
  170.  
  171. def update_github(self, id, **kwargs):
  172. pymongo.db.github.find_one_and_update({'_id': id}, update={'$set': kwargs})
  173.  
  174. def get_travis_logs(self):
  175. pass
  176.  
  177. def get_artifacts_from_db(self, instance_name):
  178. collection = pymongo.db[instance_name]
  179. docs = collection.find({})
  180. return list(map(lambda x: x, docs))
  181.  
  182. def get_all_collection(self):
  183. data = dict.fromkeys(["travis", "jenkins", "github", "jira"])
  184. for instance in data.keys():
  185. data[instance] = self.get_artifacts_from_db(instance)
  186. return data
  187.  
  188. def close(self):
  189. self.client.close()
  190.  
  191. if __name__ == "__main__":
  192. DbObj = MongoInstance()
  193. DbObj.insert_last_from_travis()
  194. # DbObj.close()
  195. # DbObj.insert_all_from_jenkins()
  196. # pymongo.db.create_collection("travis")
  197. # pymongo.db.create_collection("github")
  198. # insert_all_from_travis()
  199. #DbObj.insert_last_from_jenkins()
  200. # insert_all_from_github()
  201. # show_docs()
  202. # update_github()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement