commit 83da165365b7a6e9fda5313b4f44f4eddb6dd1c2
Author: Oleg Tsarev <zabivator@gmail.com>
Date:   Sun Aug 5 21:37:56 2012 +0400

    fix

diff --git a/tests/harness/runN.py b/tests/harness/runN.py
index 7943161..d73144c 100755
--- a/tests/harness/runN.py
+++ b/tests/harness/runN.py
@@ -9,6 +9,7 @@ import sys
 import time
 import os
 import string
+import itertools

 # Just for sanity. You could probably run more if you wanted to
 MAX_SUPPORTED_INSTANCES = 20
@@ -193,22 +194,79 @@ def shutdown(p):
   p.terminate()

 def check_pidfile():
+  RETRY_COUNT = int(os.environ.get('SCIDB_KILL_TIMEOUT', 5))
   if (os.path.exists(pidfile)):
-    f = open(pidfile, 'r')
-    for line in f:
-      pid = line.strip()
-      procfile = "/proc/" + pid + "/stat"
-      if (os.path.exists(procfile)):
-        f1 = open(procfile, 'r')
-        progname = string.split(f1.readline())[1]
-        f1.close()
-        if ( progname == "(scidb)" ) or ( dlaRunner != '0' and progname == "("+dlaRunner+")" ):
-          print "NOTE: killing old scidb instance at PID " + pid
-          subprocess.Popen(["kill", "-SIGTERM", pid], cwd=".").wait()
-          time.sleep(1)
-          subprocess.Popen(["kill", "-9", pid], cwd=".").wait()
-          while (os.path.exists(procfile)):
-            time.sleep(1)
+    def get_prog_name(pid):
+      result = None
+      # proc file name
+      pfn = "/proc/%s/stat" % pid
+      if os.path.exists(pfn):
+        # proc file
+        with open(pfn, 'r') as pf:
+          result = string.split(pf.readline())[1]
+          pf.close()
+      return result
+
+    def is_actual_pid(pid):
+      prog_name = get_prog_name(pid)
+      return prog_name == "(scidb)" or (dlaRunner != '0' and prog_name== "(%s)" % dlaRunner)
+
+    watchdog_list = filter(is_actual_pid, map(string.strip, open(pidfile, 'r')))
+
+    def get_children(pid):
+      ps = subprocess.Popen(["ps", "-o", "pid", "--no-headers", "--ppid", pid], stdout=subprocess.PIPE, cwd=".")
+      ps.wait()
+      result = []
+      for item in ps.communicate():
+        if not item is None:
+          result.append(item.strip())
+      print "children for '%s' are [%s]" % (pid, ",".join(result))
+      return result
+
+    child_list = []
+    for item_list in map(get_children, watchdog_list):
+      for item in item_list:
+        child_list.append(item)
+
+    def kill(pid_list, signal):
+      killer_list = []
+      for pid in pid_list:
+        print "NOTE: killing old scidb instance at PID %s by %s signal" % (pid, signal)
+        killer_list.append(subprocess.Popen(["kill", signal, pid], cwd="."))
+
+      time.sleep(1)
+
+      for killer in killer_list:
+        killer.wait()
+
+    kill(watchdog_list, "-SIGTERM")
+    watchdog_list = filter(is_actual_pid, watchdog_list)
+    kill(watchdog_list, "-9")
+    watchdog_list = filter(is_actual_pid, watchdog_list)
+
+    retry = RETRY_COUNT
+    while retry > 0 and len(watchdog_list):
+      print "Waiting while SciDB processes are completing, pid list [%s]" % (",".join(watchdog_list))
+      time.sleep(1)
+      watchdog_list = filter(is_actual_pid, watchdog_list)
+      retry -= 1
+
+    if len(watchdog_list):
+      sys.stderr.write('Some SciDB processes still runing: [%s]\n' % ",".join(watchdog_list))
+      exit(-1)
+
+    child_list = filter(is_actual_pid, child_list)
+
+    retry = RETRY_COUNT
+    while retry > 0 and len(child_list):
+      print "Waiting while SciDB child processes are completing, pid list [%s]" % (",".join(child_list))
+      time.sleep(1)
+      child_list = filter(is_actual_pid, child_list)
+      retry -= 1
+
+    if len(child_list):
+      sys.stderr.write("Some SciDB schild processes still runing: [%s]\n" % ",".join(watchdog_list))
+      exit(-1)

 def savepids(master_pid, instancePids):
   subprocess.Popen(["rm", "-f", pidfile], cwd=".").wait()