Guest User

Untitled

a guest
Dec 12th, 2017
89
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.65 KB | None | 0 0
  1. import argparse
  2. import os
  3. import progressbar # progressbar2
  4. import signal
  5. import time
  6.  
  7. from git import Repo, GitCommandError # gitpython
  8. from multiprocessing import Lock, Process, Queue
  9. from Queue import Empty
  10.  
  11. # Paths
  12. closest_save_path = "closest.dat"
  13. progress_save_path = ".progress"
  14.  
  15. # Multiprocessing
  16. stopping = False
  17.  
  18.  
  19. class Diff:
  20. def __init__(self, sha, count):
  21. self.sha = sha
  22. self.count = int(count)
  23.  
  24.  
  25. def save_closest(path, dictionary):
  26. f = open(path, "w")
  27. for key in dictionary:
  28. obj = dictionary[key]
  29. f.write(key + "\t" + str(obj.sha) + "\t" + str(obj.count) + "\n")
  30. f.close()
  31.  
  32.  
  33. def load_closest(path):
  34. dictionary = {}
  35. try:
  36. f = open(path, "r")
  37. lines = f.read().splitlines()
  38. for line in lines:
  39. if line != "":
  40. try:
  41. tokens = line.split("\t")
  42. dictionary[tokens[0]] = Diff(tokens[1], tokens[2])
  43. except IndexError:
  44. pass
  45. except ValueError:
  46. pass
  47. f.close()
  48. except IOError:
  49. pass
  50. return dictionary
  51.  
  52.  
  53. def check_revision(git, git_lock, path, revision):
  54. git_lock.acquire()
  55. try:
  56. diff = git.diff("--find-renames", "--numstat", "-w", "--ignore-blank-lines", revision, "--", path)
  57. finally:
  58. git_lock.release()
  59. diff_details = diff.split("\t")
  60. if len(diff_details) == 1 and diff_details[0] == "":
  61. changes = 0
  62. else:
  63. changes = int(diff_details[0]) + int(diff_details[1])
  64. return Diff(revision, changes)
  65.  
  66.  
  67. def do_check_path(queue, git, git_lock, branch, path):
  68. global stopping
  69.  
  70. git_lock.acquire()
  71. try:
  72. commits = git.log("-m", "--first-parent", "--oneline", "--follow", branch, "--", path).split("\n")
  73. finally:
  74. git_lock.release()
  75.  
  76. for i, commit in enumerate(commits):
  77. if stopping:
  78. break
  79.  
  80. sha = commit.split(" ")[0]
  81.  
  82. diff = check_revision(git, git_lock, path, sha)
  83. if diff.count == 0:
  84. git_lock.acquire()
  85. try:
  86. git.checkout(path)
  87. queue.put(["closest_rm", path])
  88. finally:
  89. git_lock.release()
  90. break
  91. queue.put(["closest_add", path, diff])
  92.  
  93. queue.put(["stop", path])
  94.  
  95.  
  96. def check_path(queue, git, git_lock, branch, path):
  97. try:
  98. do_check_path(queue, git, git_lock, branch, path)
  99. except GitCommandError:
  100. queue.put(["stop", path])
  101.  
  102.  
  103. def signal_handler(*_):
  104. global stopping
  105. stopping = True
  106.  
  107.  
  108. def main(branches, args):
  109. global stopping, progress_save_path, closest_save_path
  110.  
  111. signal.signal(signal.SIGINT, signal_handler)
  112.  
  113. queues = list()
  114.  
  115. repo = Repo(args.kernel_path)
  116. git = repo.git
  117. git_lock = Lock()
  118.  
  119. last_branch = ""
  120. last_paths = list()
  121.  
  122. closest_revisions = load_closest(closest_save_path)
  123.  
  124. try:
  125. f = open(progress_save_path, "r")
  126. lines = f.read().splitlines()
  127. try:
  128. last_branch = lines[0]
  129. last_paths = lines[1:]
  130. except IndexError:
  131. pass
  132. f.close()
  133. except IOError:
  134. pass
  135.  
  136. if last_branch != "":
  137. try:
  138. branches = branches[branches.index(last_branch):]
  139. except IndexError:
  140. pass
  141.  
  142. last_save = 0
  143.  
  144. for branch in branches:
  145. if stopping:
  146. break
  147. last_branch = branch
  148. paths = git.diff("--name-only").split("\n")
  149. n = len(paths)
  150. offset = 0
  151. for path in last_paths:
  152. try:
  153. del paths[paths.index(path)]
  154. offset += 1
  155. except ValueError:
  156. pass
  157. if n > 0 and offset < n:
  158. widgets = [
  159. progressbar.Percentage(),
  160. " (",
  161. progressbar.Counter(),
  162. " of " + str(n) + ") ",
  163. progressbar.Bar(),
  164. " ",
  165. progressbar.Timer(),
  166. " " * 50
  167. ]
  168. bar = progressbar.ProgressBar(max_value=n, widgets=widgets, term_width=150)
  169. j = offset + 1
  170. for path in paths:
  171. bar.update(j)
  172. if stopping:
  173. break
  174. if len(queues) < args.max_threads:
  175. bar.update(j)
  176. queue = Queue()
  177. Process(target=check_path, args=(queue, git, git_lock, branch, path)).start()
  178. queues.append(queue)
  179. while len(queues) == args.max_threads or (paths.index(path) == len(paths) - 1 and len(queues) > 0):
  180. bar.update(j)
  181. for queue in queues[:]:
  182. bar.update(j)
  183. try:
  184. data = queue.get(False)
  185. data_action = data[0]
  186. data_path = data[1]
  187. if data_action == "stop":
  188. if not stopping:
  189. label = " -- " + data_path
  190. widgets[7] = label + " " * (50 - len(label))
  191. j += 1
  192. last_paths.append(data_path)
  193. queues.remove(queue)
  194. if data_action == "closest_add":
  195. diff = data[2]
  196. if data_path not in closest_revisions or diff.count < closest_revisions[data_path]:
  197. closest_revisions[data_path] = diff
  198. if data_action == "closest_rm":
  199. if data_path in closest_revisions:
  200. del closest_revisions[data_path]
  201. now = time.time()
  202. if now - last_save > 60:
  203. f = open(progress_save_path, "w")
  204. f.write(last_branch)
  205. for last_path in last_paths:
  206. f.write("\n" + last_path)
  207. f.close()
  208. save_closest(closest_save_path, closest_revisions)
  209. last_save = now
  210. except Empty:
  211. pass
  212. if not stopping:
  213. bar.finish()
  214. if not stopping:
  215. last_paths = list()
  216.  
  217. if stopping and last_branch != "":
  218. f = open(progress_save_path, "w")
  219. f.write(last_branch)
  220. for path in last_paths:
  221. f.write("\n" + path)
  222. f.close()
  223.  
  224. if not stopping:
  225. os.remove(progress_save_path)
  226.  
  227. save_closest(closest_save_path, closest_revisions)
  228.  
  229.  
  230. if __name__ == "__main__":
  231. parser = argparse.ArgumentParser()
  232. parser.add_argument("-j",
  233. type=int,
  234. dest="max_threads",
  235. default=16,
  236. help="Maximum number of threads (default: 16)")
  237. parser.add_argument("-k", "--kernel_path",
  238. type=str,
  239. dest="kernel_path",
  240. required=True,
  241. help="Location of the Kernel folder")
  242.  
  243. # Branches or tags or commits
  244. revisions = [
  245. # "linaro-2.6.39/master",
  246. # "linaro-3.0/master",
  247. # "linaro-3.1/master",
  248. "common/deprecated/android-2.6.39",
  249. "common/deprecated/android-3.0",
  250. "common/deprecated/android-3.3",
  251. "common/deprecated/android-3.4-compat",
  252. "common/deprecated/android-3.4",
  253. # "kgene/master",
  254. # "krzk/master"
  255. ]
  256. main(revisions, parser.parse_args())
Add Comment
Please, Sign In to add comment