Guest User

Untitled

a guest
Dec 12th, 2017
71
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.42 KB | None | 0 0
  1. import argparse
  2. import os
  3. import progressbar # progressbar2
  4. import signal
  5. import time
  6.  
  7. from git import Repo, GitCommandError # gitpython
  8. from multiprocessing import Lock, Process, Queue
  9. from Queue import Empty
  10.  
  11. # Paths
  12. closest_save_path = "closest.dat"
  13. progress_save_path = ".progress"
  14.  
  15. # Multiprocessing
  16. stopping = False
  17.  
  18.  
  19. class Diff:
  20. def __init__(self, sha, count):
  21. self.sha = sha
  22. self.count = int(count)
  23.  
  24.  
  25. def save_closest(path, dictionary):
  26. f = open(path, "w")
  27. for key in dictionary:
  28. obj = dictionary[key]
  29. f.write(key + "\t" + str(obj.sha) + "\t" + str(obj.count) + "\n")
  30. f.close()
  31.  
  32.  
  33. def load_closest(path):
  34. dictionary = {}
  35. try:
  36. f = open(path, "r")
  37. lines = f.read().splitlines()
  38. for line in lines:
  39. if line != "":
  40. try:
  41. tokens = line.split("\t")
  42. dictionary[tokens[0]] = Diff(tokens[1], tokens[2])
  43. except IndexError:
  44. pass
  45. except ValueError:
  46. pass
  47. f.close()
  48. except IOError:
  49. pass
  50. return dictionary
  51.  
  52.  
  53. def do_check_path(queue, git, git_lock, path, sha):
  54. global stopping
  55.  
  56. git_lock.acquire()
  57. try:
  58. diff = git.diff("--numstat", "-w", "--ignore-blank-lines", path)
  59. diff_details = diff.split("\t")
  60. if len(diff_details) == 1 and diff_details[0] == "":
  61. queue.put(["closest_rm", path])
  62. queue.put(["stop", path])
  63. return
  64. changes = git.diff("-U0", "-w", "--ignore-blank-lines", path).split("\n")[4:]
  65. changes_closest = git.diff("-U0", "-w", "--ignore-blank-lines", sha, "--", path).split("\n")[4:]
  66. finally:
  67. git_lock.release()
  68.  
  69. cherrypick = True
  70.  
  71. for i, change in enumerate(changes):
  72. if stopping:
  73. break
  74. if change[0:2] == "@@":
  75. continue
  76. for j, change_closest in enumerate(changes_closest):
  77. if stopping:
  78. break
  79. if change_closest[0:2] == "@@":
  80. continue
  81. if change == change_closest:
  82. cherrypick = False
  83.  
  84. if cherrypick:
  85. git_lock.acquire()
  86. try:
  87. git.checkout(path)
  88. queue.put(["closest_rm", path])
  89. finally:
  90. git_lock.release()
  91.  
  92. queue.put(["stop", path])
  93.  
  94.  
  95. def check_path(queue, git, git_lock, path, sha):
  96. try:
  97. do_check_path(queue, git, git_lock, path, sha)
  98. except GitCommandError:
  99. queue.put(["stop", path])
  100.  
  101.  
  102. def signal_handler(*_):
  103. global stopping
  104. stopping = True
  105.  
  106.  
  107. def main(args):
  108. global stopping, progress_save_path, closest_save_path
  109.  
  110. signal.signal(signal.SIGINT, signal_handler)
  111.  
  112. queues = list()
  113.  
  114. repo = Repo(args.kernel_path)
  115. git = repo.git
  116. git_lock = Lock()
  117.  
  118. last_branch = ""
  119. last_paths = list()
  120.  
  121. closest_revisions = load_closest(closest_save_path)
  122.  
  123. try:
  124. f = open(progress_save_path, "r")
  125. lines = f.read().splitlines()
  126. try:
  127. last_paths = lines[1:]
  128. except IndexError:
  129. pass
  130. f.close()
  131. except IOError:
  132. pass
  133.  
  134. last_save = 0
  135.  
  136. paths = closest_revisions.keys()
  137. n = len(paths)
  138. offset = 0
  139. for path in last_paths:
  140. try:
  141. del paths[paths.index(path)]
  142. offset += 1
  143. except ValueError:
  144. pass
  145. if n > 0 and offset < n:
  146. widgets = [
  147. progressbar.Percentage(),
  148. " (",
  149. progressbar.Counter(),
  150. " of " + str(n) + ") ",
  151. progressbar.Bar(),
  152. " ",
  153. progressbar.Timer(),
  154. " " * 50
  155. ]
  156. bar = progressbar.ProgressBar(max_value=n, widgets=widgets, term_width=150)
  157. j = offset + 1
  158. for path in paths:
  159. bar.update(j)
  160. if stopping:
  161. break
  162. if len(queues) < args.max_threads:
  163. bar.update(j)
  164. queue = Queue()
  165. Process(target=check_path, args=(queue, git, git_lock, path, closest_revisions[path].sha)).start()
  166. queues.append(queue)
  167. while len(queues) == args.max_threads or (paths.index(path) == len(paths) - 1 and len(queues) > 0):
  168. bar.update(j)
  169. for queue in queues[:]:
  170. bar.update(j)
  171. try:
  172. data = queue.get(False)
  173. data_action = data[0]
  174. data_path = data[1]
  175. if data_action == "stop":
  176. if not stopping:
  177. label = " -- " + data_path
  178. widgets[7] = label + " " * (50 - len(label))
  179. j += 1
  180. last_paths.append(data_path)
  181. queues.remove(queue)
  182. if data_action == "closest_rm":
  183. if data_path in closest_revisions:
  184. del closest_revisions[data_path]
  185. now = time.time()
  186. if now - last_save > 10:
  187. f = open(progress_save_path, "w")
  188. for last_path in last_paths:
  189. f.write("\n" + last_path)
  190. f.close()
  191. save_closest(closest_save_path, closest_revisions)
  192. last_save = now
  193. except Empty:
  194. pass
  195. if not stopping:
  196. bar.finish()
  197. if not stopping:
  198. last_paths = list()
  199.  
  200. if stopping and last_branch != "":
  201. f = open(progress_save_path, "w")
  202. f.write(last_branch)
  203. for path in last_paths:
  204. f.write("\n" + path)
  205. f.close()
  206.  
  207. if not stopping:
  208. os.remove(progress_save_path)
  209.  
  210. save_closest(closest_save_path, closest_revisions)
  211.  
  212.  
  213. if __name__ == "__main__":
  214. parser = argparse.ArgumentParser()
  215. parser.add_argument("-j",
  216. type=int,
  217. dest="max_threads",
  218. default=16,
  219. help="Maximum number of threads (default: 16)")
  220. parser.add_argument("-k", "--kernel_path",
  221. type=str,
  222. dest="kernel_path",
  223. required=True,
  224. help="Location of the Kernel folder")
  225.  
  226. main(parser.parse_args())
Add Comment
Please, Sign In to add comment