Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from instance_factory import collect_builders
- from instance_factory import collect_bug_trackers
- import pymongo
- from tools.log_parser import LogParser
- import json
- class MongoInstance():
- def __init__(self):
- self.client = pymongo.MongoClient(
- r"mongodb://heroku_6m3l526r:7mesdarsb1n8jnfvv5lv913j4r@ds259085.mlab.com:59085/heroku_6m3l526r")
- pymongo.db = self.client.heroku_6m3l526r
- with open('Config_for_database_repositories.json', 'r') as file:
- Repositories = file.read()
- parsed__Repository = json.loads(Repositories)
- self.List_Travis_Rep = parsed__Repository.get("Travis_Repository")
- self.List_Github_Rep = parsed__Repository.get("Github_Repository")
- DB_Builders = collect_builders()
- #DB_Bugs = collect_bug_trackers()
- self.travis = DB_Builders[1]
- #self.jenkins = DB_Builders[0]
- #self.github = DB_Bugs[0]
- #self.jira = DB_Bugs[1]
- def show_docs(self):
- for docs in pymongo.db.travis.find():
- print(docs)
- print(docs['Repository'])
- def insert_all_from_jenkins(self):
- builds = self.jenkins.get_all_logs("numpy")
- collection = pymongo.db["jenkins"]
- server_name = self.jenkins.server_name
- for build in builds:
- raw = {
- "log": build.get_console(),
- "time": str(build.get_duration()),
- "status": build.get_status(),
- "author": build.get_causes()[0]["shortDescription"],
- "revision": build.get_revision(),
- "server": server_name,
- "name": build.baseurl
- }
- if not collection.find_one(raw):
- collection.insert(raw)
- def insert_last_from_travis(self):
- for rep in range(len(self.List_Travis_Rep)):
- print('Working with...', self.List_Travis_Rep[rep])
- log = self.travis.get_all_fails(self.List_Travis_Rep[rep])
- for key in log:
- print("%s -> %s" % (key, log[key]))
- '''
- Log_id = log["id"]
- Build_lanquage = log["language"]
- Job_id = log["job_id"]
- Committer_name = log["committer_name"]
- Committer_email = log["committer_email"]
- Commit_message = log["message"]
- Committed_time = log["committed_at"]
- Parsed_log = LogParser().parse_travis_log(log["content"].rstrip())
- if (Log_id == -1):
- print('Connection problem with travis')
- continue
- doc = pymongo.db.travis.find_one(
- {
- 'Log_id': Log_id
- }
- )
- if (doc):
- print("Log already put in (", Log_id, ')')
- else:
- print("Inserting log... (", Log_id, ')')
- pymongo.db.travis.insert(
- {
- 'Repository': self.List_Travis_Rep[rep],
- 'Build_language': Build_lanquage,
- 'Job_id': Job_id,
- 'Log_committer_name': Committer_name,
- 'Message': Commit_message,
- 'Log_committer_email': Committer_email,
- 'Log_committed_time': Committed_time,
- 'Log_id': Log_id,
- 'Log': Parsed_log
- }
- )
- '''
- def insert_all_from_travis(self):
- for rep in range(len(self.List_Travis_Rep)):
- print('Working with...', self.List_Travis_Rep[rep])
- for log in self.travis.get_all_fails(self.List_Travis_Rep[rep]):
- Log_id = log["id"]
- Build_lanquage = log["language"]
- Job_id = log["job_id"]
- Committer_name = log["committer_name"]
- Committer_email = log["committer_email"]
- Commit_message = log["message"]
- Committed_time = log["committed_at"]
- Parsed_log = LogParser().parse_travis_log(log["content"].rstrip())
- if (Log_id == -1):
- print('Connection problem with travis')
- continue
- doc = pymongo.db.travis.find_one(
- {
- 'Log_id': Log_id
- }
- )
- if (doc):
- print("Log already put in (", Log_id, ')')
- continue
- else:
- print("Inserting log... (", Log_id, ')')
- pymongo.db.travis.insert(
- {
- 'Repository': self.List_Travis_Rep[rep],
- 'Build_language': Build_lanquage,
- 'Job_id': Job_id,
- 'Log_committer_name': Committer_name,
- 'Message': Commit_message,
- 'Log_committer_email': Committer_email,
- 'Log_committed_time': Committed_time,
- 'Log_id': Log_id,
- 'Log': Parsed_log
- }
- )
- def insert_all_from_jira(self):
- """
- Inserts all issues from jira with credentials given in instances.json.
- Structure of issue document can be find in jira_tracker.py.
- """
- issues = self.jira.get_all_issues()
- collection = pymongo.db.jira
- for issue in issues:
- if not collection.find_one(issue):
- collection.insert(issue)
- def insert_all_from_github(self):
- for rep in range(len(self.List_Github_Rep)):
- print('Working with...', self.List_Github_Rep[rep])
- issues = self.github.get_all_issues(self.List_Github_Rep[rep])
- for issue in issues:
- closed_by_login = ''
- closed_by_id = ''
- repository_url = issue['repository_url']
- id_issue = issue['id']
- number = issue['number']
- title = issue['title'].strip()
- user_login = issue['user']['login']
- user_id = issue['user']['id']
- created_at = issue['created_at']
- updated_at = issue['updated_at']
- closed_at = issue['closed_at']
- if (issue['closed_by']):
- closed_by_login = issue['closed_by']['login']
- closed_by_id = issue['closed_by']['id']
- body = issue['body'].strip()
- doc = pymongo.db.github.find_one(
- {
- 'Issue_id': id_issue
- }
- )
- if (doc):
- print("Issue already put in (", id_issue, ')')
- continue
- else:
- print('Inserting issue ...(', id_issue, ')')
- pymongo.db.github.insert(
- {
- 'Repository': self.List_Github_Rep[rep],
- 'Repository_url': repository_url,
- 'Issue_id': id_issue,
- 'Issue_number': number,
- 'Issue_title': title,
- 'User_login': user_login,
- 'User_id': user_id,
- 'Created_at': created_at,
- 'Updated_at': updated_at,
- 'Closed_at': closed_at,
- 'Closed_by_login': closed_by_login,
- 'Closed_by_id': closed_by_id,
- 'Body': body,
- }
- )
- def update_jenkins(self, id, **kwargs):
- pymongo.db.jenkins.find_one_and_update({'_id': id}, update={'$set': kwargs})
- def update_jira(self, id, **kwargs):
- pymongo.db.jira.find_one_and_update({'_id': id}, update={'$set': kwargs})
- def update_travis(self, id, ** kwargs):
- pymongo.db.travis.find_one_and_update({'_id': id}, update={'$set': kwargs})
- def update_github(self, id, **kwargs):
- pymongo.db.github.find_one_and_update({'_id': id}, update={'$set': kwargs})
- def get_travis_logs(self):
- pass
- def get_artifacts_from_db(self, instance_name):
- collection = pymongo.db[instance_name]
- docs = collection.find({})
- return list(map(lambda x: x, docs))
- def get_all_collection(self):
- data = dict.fromkeys(["travis", "jenkins", "github", "jira"])
- for instance in data.keys():
- data[instance] = self.get_artifacts_from_db(instance)
- return data
- def close(self):
- self.client.close()
- if __name__ == "__main__":
- DbObj = MongoInstance()
- DbObj.insert_last_from_travis()
- # DbObj.close()
- # DbObj.insert_all_from_jenkins()
- # pymongo.db.create_collection("travis")
- # pymongo.db.create_collection("github")
- # insert_all_from_travis()
- #DbObj.insert_last_from_jenkins()
- # insert_all_from_github()
- # show_docs()
- # update_github()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement