[marve@hpc1 ~]$  qsub -lnodes=2:ppn=2,walltime=1000 -I
qsub: waiting for job 8.hpc1.cc.uit.no to start
qsub: job 8.hpc1.cc.uit.no ready

[marve@compute-0-2 ~]$ cd rpmbuild/BUILD/mpiexec-0.84/
[marve@compute-0-2 mpiexec-0.84]$ gdb ./mpiexec
GNU gdb (GDB) Red Hat Enterprise Linux (7.2-56.el6)
Copyright (C) 2010 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /home/marve/rpmbuild/BUILD/mpiexec-0.84/mpiexec...done.
(gdb) set args /home/marve/hello.mpich2.x
(gdb) run
Starting program: /home/marve/rpmbuild/BUILD/mpiexec-0.84/mpiexec /home/marve/hello.mpich2.x
[Thread debugging using libthread_db enabled]
Detaching after fork from child process 20621.
hello from 3/4 hostname compute-0-1.local pid 20155 with 0 args:
hello from 0/4 hostname compute-0-2.local pid 20622 with 0 args:
hello from 1/4 hostname compute-0-2.local pid 20623 with 0 args:
hello from 2/4 hostname compute-0-1.local pid 20148 with 0 args:
^C
Program received signal SIGINT, Interrupt.
0x0000003fd70ac3a0 in __nanosleep_nocancel () from /lib64/libc.so.6
Missing separate debuginfos, use: debuginfo-install glibc-2.12-1.80.el6_3.5.x86_64 keyutils-libs-1.4-4.el6.x86_64 krb5-libs-1.9-33.el6_3.3.x86_64 libcom_err-1.41.12-12.el6.x86_64 libselinux-2.0.94-5.3.el6.x86_64 libxml2-2.7.6-8.el6_3.3.x86_64 openssl-1.0.0-25.el6_3.1.x86_64 zlib-1.2.3-27.el6.x86_64
(gdb) bt
#0  0x0000003fd70ac3a0 in __nanosleep_nocancel () from /lib64/libc.so.6
#1  0x0000003fd70e1574 in usleep () from /lib64/libc.so.6
#2  0x0000000000408f17 in wait_task_start () at start_tasks.c:920
#3  0x0000000000408d44 in start_tasks (spawn=0) at start_tasks.c:843
#4  0x00000000004044a9 in main (argc=1, argv=0x7fffffffdf50) at mpiexec.c:825
(gdb) set args -comm none uname -a
(gdb) run
The program being debugged has been started already.
Start it from the beginning? (y or n) y
Starting program: /home/marve/rpmbuild/BUILD/mpiexec-0.84/mpiexec -comm none uname -a
[Thread debugging using libthread_db enabled]
Detaching after fork from child process 20628.
Linux compute-0-2.local 2.6.32-279.11.1.el6.x86_64 #1 SMP Tue Oct 16 15:57:10 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux
Linux compute-0-2.local 2.6.32-279.11.1.el6.x86_64 #1 SMP Tue Oct 16 15:57:10 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux
Linux compute-0-1.local 2.6.32-279.11.1.el6.x86_64 #1 SMP Tue Oct 16 15:57:10 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux
Linux compute-0-1.local 2.6.32-279.11.1.el6.x86_64 #1 SMP Tue Oct 16 15:57:10 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux
^C
Program received signal SIGINT, Interrupt.
0x0000003fd70ac3a0 in __nanosleep_nocancel () from /lib64/libc.so.6
(gdb) bt
#0  0x0000003fd70ac3a0 in __nanosleep_nocancel () from /lib64/libc.so.6
#1  0x0000003fd70e1574 in usleep () from /lib64/libc.so.6
#2  0x0000000000408f17 in wait_task_start () at start_tasks.c:920
#3  0x0000000000408d44 in start_tasks (spawn=0) at start_tasks.c:843
#4  0x00000000004044a9 in main (argc=2, argv=0x7fffffffdf40) at mpiexec.c:825
(gdb) list
657
658     static void show_exit_statuses(void);
659
660     int
661     main(int argc, const char *argv[])
662     {
663         int i, j, jmp_return, ret;
664         struct passwd *pswd_tmp;
665
666         set_progname(argc, argv);
(gdb) up
#1  0x0000003fd70e1574 in usleep () from /lib64/libc.so.6
(gdb) up
#2  0x0000000000408f17 in wait_task_start () at start_tasks.c:920
920                 usleep(200000);
(gdb) list
915
916             ep = poll_event();
917             if (ep)
918                 dispatch_event(ep);
919             else
920                 usleep(200000);
921
922             if (ret || numtasks_waiting_start_entry != numtasks_waiting_start)
923                 break;
924         }
(gdb)