Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python
- #-*- coding: utf-8 -*-
- import os
- import sys
- import datetime
- import subprocess
- import time
- import gevent.pool
- import gevent.monkey
- from gevent import Timeout
- gevent.monkey.patch_all()
- from multiprocessing import Process
- def shell_cmd(cmd, system=False):
- if system:
- return '', os.system(cmd)
- return subprocess.Popen(cmd, shell=not isinstance(cmd, list),
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE).communicate()
- def slice_list(ls, n):
- if not isinstance(ls, list) or not isinstance(n, int):
- return []
- ls_len = len(ls)
- if n <= 0 or 0 == ls_len:
- return []
- if n > ls_len:
- return []
- elif n == ls_len:
- return [[i] for i in ls]
- else:
- j = ls_len / n
- k = ls_len % n
- ls_return = []
- for i in xrange(0, (n-1) * j, j):
- ls_return.append(ls[i:i+j])
- ls_return.append(ls[(n-1)*j:])
- return ls_return
- def pull_worker(image):
- try:
- with Timeout(600):
- cmd = 'docker pull {0}'.format(image)
- shell_cmd(cmd)
- out, err = shell_cmd('docker images {0}'.format(image))
- if not err:
- out, err = shell_cmd('docker rmi {0}'.format(image))
- if err:
- print "[ERROR docker rmi] {0} {1}".format(image, err)
- with open('good_image.txt','a') as goodImageFile:
- goodImageFile.write(image + '\n')
- else:
- with open('error_image.txt','a') as timeout_file:
- timeout_file.write(image + " {0}".format(err) + '\n')
- except Timeout:
- with open('timeout_image.txt','a') as timeout_file:
- timeout_file.write(image + '\n')
- if __name__ == "__main__":
- start = time.time()
- all_task_list = []
- if len(sys.argv) == 4:
- pass
- else:
- print "Need three params:\n# 1 File\n# 2 Process numbers\n# 3 Parallels numbers for each process"
- sys.exit(-1)
- _, file, coreNum, poolNum = sys.argv
- def each_process(task_object_list):
- pool = gevent.pool.Pool(int(poolNum))
- pool.map(pull_worker, task_object_list)
- stop = time.time()
- elapsed = stop - start
- print "End precess with {0} s".format(elapsed)
- with open(file) as f:
- for line in f:
- line = line.strip()
- all_task_list.append(line)
- print "All task: {0}".format(len(all_task_list))
- for sliced_task_list in slice_list(all_task_list, int(coreNum)):
- print "Start process with tasks: {0}".format(len(sliced_task_list))
- p = Process(target=each_process, args=(sliced_task_list,))
- p.start()
Add Comment
Please, Sign In to add comment