Guest User

Untitled

a guest
Jan 19th, 2018
103
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.21 KB | None | 0 0
  1. #!/usr/bin/python
  2.  
  3. import swiftclient
  4. import pprint
  5. import Queue
  6. import threading
  7. import time
  8. import timeit
  9. import keystoneclient
  10. import ConfigParser
  11. import sys
  12.  
  13. #global variant from config file
  14. admin_user = ''
  15. admin_pass = ''
  16. source_auth_url = ''
  17. source_swift_base = ''
  18. destination_auth_url = ''
  19. destination_swift_base = ''
  20. n_tenant_threads = 1
  21.  
  22. #local variant use for code only
  23. pp = pprint.PrettyPrinter(indent=4)
  24. exitFlag = 0
  25.  
  26. def get_Config():
  27. global admin_user
  28. global admin_pass
  29. global source_auth_url
  30. global destination_auth_url
  31. global n_tenant_threads
  32. global account_list
  33. global source_swift_base
  34. global destination_swift_base
  35. global compare_object
  36.  
  37. config = ConfigParser.ConfigParser()
  38. config.read(r'compare-clusters.conf')
  39.  
  40. admin_user = config.get('global', 'admin_user')
  41. admin_pass = config.get('global', 'admin_pass')
  42. source_auth_url = config.get('global', 'source_auth_url')
  43. destination_auth_url = config.get('global', 'destination_auth_url')
  44. n_tenant_threads = config.getint('global', 'n_tenant_threads')
  45. account_list = config.get('global', 'account_list')
  46. source_swift_base = config.get('global', 'source_swift_base')
  47. destination_swift_base = config.get('global', 'destination_swift_base')
  48. compare_object = config.getboolean('global', 'compare_object')
  49.  
  50. class tenantThread(threading.Thread):
  51. def __init__(
  52. self,
  53. threadID,
  54. name,
  55. q,
  56. l,
  57. ):
  58. threading.Thread.__init__(self)
  59. self.threadID = threadID
  60. self.name = name
  61. self.q = q
  62. self.l = l
  63. def run(self):
  64. process_tenant_thread(self.name, self.q, self.l)
  65.  
  66. def process_tenant_thread(threadName, q, l):
  67. while not exitFlag:
  68. l.acquire()
  69. if not q.empty():
  70. data = q.get()
  71. l.release()
  72. #print '%s scanning Tenant %s' % (threadName, data)
  73. process_tenant(data)
  74. else:
  75. l.release()
  76. time.sleep(1)
  77.  
  78.  
  79. def make_tenant_threadlist(int_tenant_threads):
  80. threadList = []
  81. for i in range(1, int_tenant_threads + 1):
  82. threadList.append('tenant Thread-' + str(i))
  83.  
  84. return threadList
  85.  
  86. def process_tenant(data):
  87. print data
  88. global admin_user
  89. global admin_pass
  90. global source_auth_url
  91. global destination_auth_url
  92. global account_list
  93. global source_swift_base
  94. global destination_swift_base
  95. global compare_object
  96.  
  97. try:
  98. # get source cluster connection by tenant
  99. swift_s = swiftclient.client.Connection(authurl=source_auth_url, user=admin_user, key=admin_pass, tenant_name=admin_user, auth_version='2.0',os_options={'object_storage_url': source_swift_base+data[1]})
  100. # get destination cluster connection by tenant
  101. swift_d = swiftclient.client.Connection(authurl=destination_auth_url, user=admin_user, key=admin_pass, tenant_name=admin_user, auth_version='2.0',os_options={'object_storage_url': destination_swift_base+data[1]})
  102. resp_headers, containers = swift_s.get_account()
  103. for c in containers:
  104. bolfoundcontainer = find_container(swift_d, c['name'])
  105. if bolfoundcontainer:
  106. print "source account: " + data[1] + " has container name: " + c['name'] + " found in destination: " + str(bolfoundcontainer)
  107. if compare_object and bolfoundcontainer:
  108. resp_headers, objects = swift_s.get_container(c['name'])
  109. for o in objects:
  110. bolfoundobject = find_object(swift_d, c['name'], o['name'], o['hash'])
  111. if bolfoundobject:
  112. print "source account: " + data[1] + " has container name: " + c['name'] + " and object name: " + o['name'] + " & hash: " + o['hash'] + " found in destination: " + str(bolfoundobject)
  113. swift_s.close()
  114. swift_d.close()
  115. except Exception, err:
  116. print data[1] + ": " + str(err)
  117.  
  118. def find_container(swift_d, container_name):
  119. bolcheck = False
  120. try:
  121. resp_headers, containers = swift_d.get_account()
  122. for c in containers:
  123. if c['name'] == container_name:
  124. bolcheck = True
  125. except Exception:
  126. sys.exc_clear()
  127. return bolcheck
  128.  
  129. def find_object(swift_d, container_name, object_name, object_hash):
  130. bolcheck = False
  131. try:
  132. resp_headers, objects = swift_d.get_container(container_name)
  133. for o in objects:
  134. if ( o['name'] == object_name ) and ( o['hash'] == object_hash ):
  135. bolcheck = True
  136. except Exception:
  137. sys.exc_clear()
  138. return bolcheck
  139.  
  140. def get_tenant_list(account_list):
  141. with open(account_list) as f:
  142. lines = f.readlines()
  143. tenant_list = [x.strip() for x in lines]
  144. return tenant_list
  145.  
  146. def main():
  147. global n_tenant_threads
  148. global admin_user
  149. global admin_pass
  150. global source_auth_url
  151. global destination_auth_url
  152. global account_list
  153.  
  154. start = timeit.default_timer()
  155.  
  156. # Get Config
  157. get_Config()
  158.  
  159. # Get Tenant list
  160. tenant_list = get_tenant_list(account_list)
  161. tenantThreadList = make_tenant_threadlist(n_tenant_threads)
  162.  
  163. # Generate tenant dictionary by tennant list
  164. namedict = {}
  165. namecount = 1
  166. for tenant in tenant_list:
  167. namedict[namecount] = tenant
  168. namecount += 1
  169.  
  170. # Queue # = thread list
  171. queueLock = threading.Lock()
  172. workQueue = Queue.Queue(0) # queue size is infinite
  173. threads = []
  174. threadID = 1
  175.  
  176.  
  177. # Create new threads
  178. for tName in tenantThreadList:
  179. thread = tenantThread(threadID, tName, workQueue, queueLock)
  180. thread.start()
  181. threads.append(thread)
  182. threadID += 1
  183.  
  184. # Fill the queue
  185. queueLock.acquire()
  186.  
  187. # for word in nameList:
  188. for word in namedict.iteritems():
  189. workQueue.put(word)
  190. queueLock.release()
  191.  
  192. # Wait for queue to empty
  193. while not workQueue.empty():
  194. pass
  195.  
  196. # Notify threads it's time to exit
  197. global exitFlag
  198. exitFlag = 1
  199.  
  200. # Wait for all threads to complete
  201. for t in threads:
  202. t.join()
  203.  
  204. print 'Exiting Main Thread, Time Cost: %s' \
  205. % (timeit.default_timer() - start)
  206.  
  207. if __name__ == '__main__':
  208. try:
  209. main()
  210. except (KeyboardInterrupt):
  211. print "Abort, Got Keyboard Interrupt !"
  212. sys.exit(1)
Add Comment
Please, Sign In to add comment