Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import argparse
- import os
- import progressbar # progressbar2
- import signal
- import time
- from git import Repo, GitCommandError # gitpython
- from multiprocessing import Lock, Process, Queue
- from Queue import Empty
- # Paths
- closest_save_path = "closest.dat"
- progress_save_path = ".progress"
- # Multiprocessing
- stopping = False
- class Diff:
- def __init__(self, sha, count):
- self.sha = sha
- self.count = int(count)
- def save_closest(path, dictionary):
- f = open(path, "w")
- for key in dictionary:
- obj = dictionary[key]
- f.write(key + "\t" + str(obj.sha) + "\t" + str(obj.count) + "\n")
- f.close()
- def load_closest(path):
- dictionary = {}
- try:
- f = open(path, "r")
- lines = f.read().splitlines()
- for line in lines:
- if line != "":
- try:
- tokens = line.split("\t")
- dictionary[tokens[0]] = Diff(tokens[1], tokens[2])
- except IndexError:
- pass
- except ValueError:
- pass
- f.close()
- except IOError:
- pass
- return dictionary
- def check_revision(git, git_lock, path, revision):
- git_lock.acquire()
- try:
- diff = git.diff("--find-renames", "--numstat", "-w", "--ignore-blank-lines", revision, "--", path)
- finally:
- git_lock.release()
- diff_details = diff.split("\t")
- if len(diff_details) == 1 and diff_details[0] == "":
- changes = 0
- else:
- changes = int(diff_details[0]) + int(diff_details[1])
- return Diff(revision, changes)
- def do_check_path(queue, git, git_lock, branch, path):
- global stopping
- git_lock.acquire()
- try:
- commits = git.log("-m", "--first-parent", "--oneline", "--follow", branch, "--", path).split("\n")
- finally:
- git_lock.release()
- for i, commit in enumerate(commits):
- if stopping:
- break
- sha = commit.split(" ")[0]
- diff = check_revision(git, git_lock, path, sha)
- if diff.count == 0:
- git_lock.acquire()
- try:
- git.checkout(path)
- queue.put(["closest_rm", path])
- finally:
- git_lock.release()
- break
- queue.put(["closest_add", path, diff])
- queue.put(["stop", path])
- def check_path(queue, git, git_lock, branch, path):
- try:
- do_check_path(queue, git, git_lock, branch, path)
- except GitCommandError:
- queue.put(["stop", path])
- def signal_handler(*_):
- global stopping
- stopping = True
- def main(branches, args):
- global stopping, progress_save_path, closest_save_path
- signal.signal(signal.SIGINT, signal_handler)
- queues = list()
- repo = Repo(args.kernel_path)
- git = repo.git
- git_lock = Lock()
- last_branch = ""
- last_paths = list()
- closest_revisions = load_closest(closest_save_path)
- try:
- f = open(progress_save_path, "r")
- lines = f.read().splitlines()
- try:
- last_branch = lines[0]
- last_paths = lines[1:]
- except IndexError:
- pass
- f.close()
- except IOError:
- pass
- if last_branch != "":
- try:
- branches = branches[branches.index(last_branch):]
- except IndexError:
- pass
- last_save = 0
- for branch in branches:
- if stopping:
- break
- last_branch = branch
- paths = git.diff("--name-only").split("\n")
- n = len(paths)
- offset = 0
- for path in last_paths:
- try:
- del paths[paths.index(path)]
- offset += 1
- except ValueError:
- pass
- if n > 0 and offset < n:
- widgets = [
- progressbar.Percentage(),
- " (",
- progressbar.Counter(),
- " of " + str(n) + ") ",
- progressbar.Bar(),
- " ",
- progressbar.Timer(),
- " " * 50
- ]
- bar = progressbar.ProgressBar(max_value=n, widgets=widgets, term_width=150)
- j = offset + 1
- for path in paths:
- bar.update(j)
- if stopping:
- break
- if len(queues) < args.max_threads:
- bar.update(j)
- queue = Queue()
- Process(target=check_path, args=(queue, git, git_lock, branch, path)).start()
- queues.append(queue)
- while len(queues) == args.max_threads or (paths.index(path) == len(paths) - 1 and len(queues) > 0):
- bar.update(j)
- for queue in queues[:]:
- bar.update(j)
- try:
- data = queue.get(False)
- data_action = data[0]
- data_path = data[1]
- if data_action == "stop":
- if not stopping:
- label = " -- " + data_path
- widgets[7] = label + " " * (50 - len(label))
- j += 1
- last_paths.append(data_path)
- queues.remove(queue)
- if data_action == "closest_add":
- diff = data[2]
- if data_path not in closest_revisions or diff.count < closest_revisions[data_path]:
- closest_revisions[data_path] = diff
- if data_action == "closest_rm":
- if data_path in closest_revisions:
- del closest_revisions[data_path]
- now = time.time()
- if now - last_save > 60:
- f = open(progress_save_path, "w")
- f.write(last_branch)
- for last_path in last_paths:
- f.write("\n" + last_path)
- f.close()
- save_closest(closest_save_path, closest_revisions)
- last_save = now
- except Empty:
- pass
- if not stopping:
- bar.finish()
- if not stopping:
- last_paths = list()
- if stopping and last_branch != "":
- f = open(progress_save_path, "w")
- f.write(last_branch)
- for path in last_paths:
- f.write("\n" + path)
- f.close()
- if not stopping:
- os.remove(progress_save_path)
- save_closest(closest_save_path, closest_revisions)
- if __name__ == "__main__":
- parser = argparse.ArgumentParser()
- parser.add_argument("-j",
- type=int,
- dest="max_threads",
- default=16,
- help="Maximum number of threads (default: 16)")
- parser.add_argument("-k", "--kernel_path",
- type=str,
- dest="kernel_path",
- required=True,
- help="Location of the Kernel folder")
- # Branches or tags or commits
- revisions = [
- # "linaro-2.6.39/master",
- # "linaro-3.0/master",
- # "linaro-3.1/master",
- "common/deprecated/android-2.6.39",
- "common/deprecated/android-3.0",
- "common/deprecated/android-3.3",
- "common/deprecated/android-3.4-compat",
- "common/deprecated/android-3.4",
- # "kgene/master",
- # "krzk/master"
- ]
- main(revisions, parser.parse_args())
Add Comment
Please, Sign In to add comment