Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/python
- import swiftclient
- import pprint
- import Queue
- import threading
- import time
- import timeit
- import keystoneclient
- import ConfigParser
- import sys
- #global variant from config file
- admin_user = ''
- admin_pass = ''
- source_auth_url = ''
- source_swift_base = ''
- destination_auth_url = ''
- destination_swift_base = ''
- n_tenant_threads = 1
- #local variant use for code only
- pp = pprint.PrettyPrinter(indent=4)
- exitFlag = 0
- def get_Config():
- global admin_user
- global admin_pass
- global source_auth_url
- global destination_auth_url
- global n_tenant_threads
- global account_list
- global source_swift_base
- global destination_swift_base
- global compare_object
- config = ConfigParser.ConfigParser()
- config.read(r'compare-clusters.conf')
- admin_user = config.get('global', 'admin_user')
- admin_pass = config.get('global', 'admin_pass')
- source_auth_url = config.get('global', 'source_auth_url')
- destination_auth_url = config.get('global', 'destination_auth_url')
- n_tenant_threads = config.getint('global', 'n_tenant_threads')
- account_list = config.get('global', 'account_list')
- source_swift_base = config.get('global', 'source_swift_base')
- destination_swift_base = config.get('global', 'destination_swift_base')
- compare_object = config.getboolean('global', 'compare_object')
- class tenantThread(threading.Thread):
- def __init__(
- self,
- threadID,
- name,
- q,
- l,
- ):
- threading.Thread.__init__(self)
- self.threadID = threadID
- self.name = name
- self.q = q
- self.l = l
- def run(self):
- process_tenant_thread(self.name, self.q, self.l)
- def process_tenant_thread(threadName, q, l):
- while not exitFlag:
- l.acquire()
- if not q.empty():
- data = q.get()
- l.release()
- #print '%s scanning Tenant %s' % (threadName, data)
- process_tenant(data)
- else:
- l.release()
- time.sleep(1)
- def make_tenant_threadlist(int_tenant_threads):
- threadList = []
- for i in range(1, int_tenant_threads + 1):
- threadList.append('tenant Thread-' + str(i))
- return threadList
- def process_tenant(data):
- print data
- global admin_user
- global admin_pass
- global source_auth_url
- global destination_auth_url
- global account_list
- global source_swift_base
- global destination_swift_base
- global compare_object
- try:
- # get source cluster connection by tenant
- swift_s = swiftclient.client.Connection(authurl=source_auth_url, user=admin_user, key=admin_pass, tenant_name=admin_user, auth_version='2.0',os_options={'object_storage_url': source_swift_base+data[1]})
- # get destination cluster connection by tenant
- swift_d = swiftclient.client.Connection(authurl=destination_auth_url, user=admin_user, key=admin_pass, tenant_name=admin_user, auth_version='2.0',os_options={'object_storage_url': destination_swift_base+data[1]})
- resp_headers, containers = swift_s.get_account()
- for c in containers:
- bolfoundcontainer = find_container(swift_d, c['name'])
- if bolfoundcontainer:
- print "source account: " + data[1] + " has container name: " + c['name'] + " found in destination: " + str(bolfoundcontainer)
- if compare_object and bolfoundcontainer:
- resp_headers, objects = swift_s.get_container(c['name'])
- for o in objects:
- bolfoundobject = find_object(swift_d, c['name'], o['name'], o['hash'])
- if bolfoundobject:
- print "source account: " + data[1] + " has container name: " + c['name'] + " and object name: " + o['name'] + " & hash: " + o['hash'] + " found in destination: " + str(bolfoundobject)
- swift_s.close()
- swift_d.close()
- except Exception, err:
- print data[1] + ": " + str(err)
- def find_container(swift_d, container_name):
- bolcheck = False
- try:
- resp_headers, containers = swift_d.get_account()
- for c in containers:
- if c['name'] == container_name:
- bolcheck = True
- except Exception:
- sys.exc_clear()
- return bolcheck
- def find_object(swift_d, container_name, object_name, object_hash):
- bolcheck = False
- try:
- resp_headers, objects = swift_d.get_container(container_name)
- for o in objects:
- if ( o['name'] == object_name ) and ( o['hash'] == object_hash ):
- bolcheck = True
- except Exception:
- sys.exc_clear()
- return bolcheck
- def get_tenant_list(account_list):
- with open(account_list) as f:
- lines = f.readlines()
- tenant_list = [x.strip() for x in lines]
- return tenant_list
- def main():
- global n_tenant_threads
- global admin_user
- global admin_pass
- global source_auth_url
- global destination_auth_url
- global account_list
- start = timeit.default_timer()
- # Get Config
- get_Config()
- # Get Tenant list
- tenant_list = get_tenant_list(account_list)
- tenantThreadList = make_tenant_threadlist(n_tenant_threads)
- # Generate tenant dictionary by tennant list
- namedict = {}
- namecount = 1
- for tenant in tenant_list:
- namedict[namecount] = tenant
- namecount += 1
- # Queue # = thread list
- queueLock = threading.Lock()
- workQueue = Queue.Queue(0) # queue size is infinite
- threads = []
- threadID = 1
- # Create new threads
- for tName in tenantThreadList:
- thread = tenantThread(threadID, tName, workQueue, queueLock)
- thread.start()
- threads.append(thread)
- threadID += 1
- # Fill the queue
- queueLock.acquire()
- # for word in nameList:
- for word in namedict.iteritems():
- workQueue.put(word)
- queueLock.release()
- # Wait for queue to empty
- while not workQueue.empty():
- pass
- # Notify threads it's time to exit
- global exitFlag
- exitFlag = 1
- # Wait for all threads to complete
- for t in threads:
- t.join()
- print 'Exiting Main Thread, Time Cost: %s' \
- % (timeit.default_timer() - start)
- if __name__ == '__main__':
- try:
- main()
- except (KeyboardInterrupt):
- print "Abort, Got Keyboard Interrupt !"
- sys.exit(1)
Add Comment
Please, Sign In to add comment