View difference between Paste ID: kFC7HsdJ and v6KSp3e8
SHOW: | | - or go back to the newest paste.
1
#!/usr/bin/python
2
# -*- coding: utf-8 -*-
3
4
import os
5
import sys
6
import time
7
8
from multiprocessing import Process, cpu_count
9
10
#-----------------------------------------------------------------------
11
def log(msg):
12
  print "[%s %d] %s" % (time.asctime(), os.getpid(), msg)
13
14
#-----------------------------------------------------------------------
15
def dummy_process():
16
  import random
17
  t = random.randint(0, 2)
18
  log("Sleeping %d second(s)..." % t)
19
  time.sleep(t)
20
  log("Done, bye")
21
22
#-----------------------------------------------------------------------
23
def process_manager(total_procs, target, wait_time=0.2):
24
  """ Always maintain a total of @total_procs running @target and
25
      waiting for each thread to finish @wait_time second(s). """
26
  procs = []
27
  log("Maximum number of processes in pool is %d" % total_procs)
28
  while 1:
29
    if len(procs) < total_procs:
30
      log("Starting process %d" % (len(procs)+1))
31
      p = Process(target=target)
32
      p.start()
33
      procs.append(p)
34
      log("Total of %d process(es) started" % len(procs))
35
    else:
36
      i = 0
37
      for p in list(procs):
38
        p.join(wait_time)
39
        if not p.is_alive():
40
          log("Process finished, deleting and starting a new one...")
41
          del procs[i]
42
          continue
43
        i += 1
44
45
if __name__ == "__main__":
46
  total_procs = cpu_count()*5
47-
  manage_pool(total_procs, dummy_process)
47+
  process_manager(total_procs, dummy_process)