dmtcp_launch

[slurm@slurm-master ~]$  dmtcp_launch --rm srun -n 2 mpiLoop 50
[6365] TRACE at dmtcp_launch.cpp:419 in main; REASON='dmtcp_launch starting new program:'
     argv[0] = srun
[6365] TRACE at dmtcp_launch.cpp:433 in main; REASON='setting DMTCP_CHECKPOINT_DIR'
     ckptDir = /home/slurm
[6365] TRACE at coordinatorapi.cpp:586 in connectToCoordOnStartup; REASON='sending coordinator handshake'
     UniquePid::ThisProcess() = 6db90f3d5a9dd200-6365-546a1de8
[6365] TRACE at coordinatorapi.cpp:593 in connectToCoordOnStartup; REASON='Got virtual pid from coordinator'
     hello_remote.virtualPid = 40000
[6365] TRACE at shareddata.cpp:189 in initialize; REASON='Shared area mapped'
     sharedDataHeader = 0x7f3eccb11000
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at jfilesystem.cpp:273 in FindHelperUtility; REASON='JALIB_UTILITY_DIR was set:  using it'
[6365] TRACE at dmtcp_launch.cpp:707 in setLDPreloadLibs; REASON='getting value of LD_PRELOAD'
     getenv("LD_PRELOAD") = /usr/local/bin/../lib/dmtcp/libdmtcp_batch-queue.so:/usr/local/bin/../lib/dmtcp/libdmtcp_alloc.so:/usr/local/bin
/../lib/dmtcp/libdmtcp_dl.so:/usr/local/bin/../lib/dmtcp/libdmtcp_ipc.so:/usr/local/bin/../lib/dmtcp/libdmtcp.so:/usr/local/bin/../lib/dmtcp
/libdmtcp_pid.so:
     preloadLibs = /usr/local/bin/../lib/dmtcp/libdmtcp_batch-queue.so:/usr/local/bin/../lib/dmtcp/libdmtcp_alloc.so:/usr/local/bin/../lib/d
mtcp/libdmtcp_dl.so:/usr/local/bin/../lib/dmtcp/libdmtcp_ipc.so:/usr/local/bin/../lib/dmtcp/libdmtcp.so:/usr/local/bin/../lib/dmtcp/libdmtcp
_pid.so:
     preloadLibs32 = libdmtcp_batch-queue.so:libdmtcp_alloc.so:libdmtcp_dl.so:libdmtcp_ipc.so:libdmtcp.so:libdmtcp_pid.so:
[40000] TRACE at dmtcpworker.cpp:233 in prepareLogAndProcessdDataFromSerialFile; REASON='Root of processes tree'
[40000] TRACE at dmtcpworker.cpp:278 in DmtcpWorker; REASON='libdmtcp.so:  Running '
     jalib::Filesystem::GetProgramName() = srun
     getenv ("LD_PRELOAD") = /usr/local/bin/../lib/dmtcp/libdmtcp_batch-queue.so:/usr/local/bin/../lib/dmtcp/libdmtcp_alloc.so:/usr/local/bi
n/../lib/dmtcp/libdmtcp_dl.so:/usr/local/bin/../lib/dmtcp/libdmtcp_ipc.so:/usr/local/bin/../lib/dmtcp/libdmtcp.so:/usr/local/bin/../lib/dmtc
p/libdmtcp_pid.so:
[40000] TRACE at dmtcpworker.cpp:96 in restoreUserLDPRELOAD; REASON='LD_PRELOAD'
     preload =
     userPreload = [40000] TRACE at coordinatorapi.cpp:164 in init; REASON='Informing coordinator of new process'
     UniquePid::ThisProcess() = 6db90f3d5a9dd200-40000-546a1de8
[40000] TRACE at processinfo.cpp:174 in growStack; REASON='Original stack area'
     (void*)area.addr = 0x7fffe5c45000
     area.size = 86016
[40000] TRACE at processinfo.cpp:196 in growStack; REASON='New stack size'
     (void*)area.addr = 0x7fffe526a000
     area.size = 10420224
[40000] TRACE at shareddata.cpp:189 in initialize; REASON='Shared area mapped'
     sharedDataHeader = 0x7fa89e561000
[40000] TRACE at fileconnlist.cpp:250 in scanForPreExisting; REASON='scanning pre-existing device'
     fd = 0
     device = /dev/pts/1
[40000] TRACE at fileconnection.cpp:252 in PtyConnection; REASON='creating CTTY connection'
     _ptsName = /dev/pts/1
     _virtPtsName = /dev/pts/v0
[40000] TRACE at fileconnlist.cpp:250 in scanForPreExisting; REASON='scanning pre-existing device'
     fd = 1
     device = /dev/pts/1
[40000] TRACE at fileconnlist.cpp:250 in scanForPreExisting; REASON='scanning pre-existing device'
     fd = 2
     device = /dev/pts/1
[40000] TRACE at socketconnlist.cpp:150 in scanForPreExisting; REASON='scanning pre-existing device'
     fd = 0
     device = /dev/pts/1
[40000] TRACE at socketconnlist.cpp:150 in scanForPreExisting; REASON='scanning pre-existing device'
     fd = 1
     device = /dev/pts/1
[40000] TRACE at socketconnlist.cpp:150 in scanForPreExisting; REASON='scanning pre-existing device'
     fd = 2
     device = /dev/pts/1
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6365
     th->virtual_tid = 40000
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6368
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40002
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6368
     th->virtual_tid = 40002
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40002
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 3
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99005)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 3
     optname = 2
[40000] TRACE at socketwrappers.cpp:141 in bind; REASON='bind'
     sockfd = 3
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99005)
[40000] TRACE at socketwrappers.cpp:159 in listen; REASON='listen'
     sockfd = 3
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99005)
     backlog = 4096
[40000] TRACE at threadlist.cpp:318 in checkpointhread; REASON='after sigsetjmp/getcontext'
     curThread->tid = 6368
     curThread->virtual_tid = 40002
     curThread->saved_sp = 0x7fa89e55d790
[40000] TRACE at threadlist.cpp:334 in checkpointhread; REASON='before callbackSleepBetweenCheckpoint(0)'
[40000] TRACE at dmtcpworker.cpp:468 in waitForStage1Suspend; REASON='running'
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for SUSPEND message'
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 4
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99006)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 4
     optname = 2
[40000] TRACE at socketwrappers.cpp:141 in bind; REASON='bind'
     sockfd = 4
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99006)
[40000] TRACE at socketwrappers.cpp:159 in listen; REASON='listen'
     sockfd = 4
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99006)
     backlog = 1024
[40000] TRACE at miscwrappers.cpp:120 in pipe; REASON='promoting pipe() to socketpair()'
[40000] TRACE at socketwrappers.cpp:271 in socketpair; REASON='socketpair()'
     sv[0] = 5
     sv[1] = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99007)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6369
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40004
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6369
     th->virtual_tid = 40004
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40004
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 7
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99009)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 7
     optname = 2
[40000] TRACE at socketwrappers.cpp:141 in bind; REASON='bind'
     sockfd = 7
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99009)
[40000] TRACE at socketwrappers.cpp:159 in listen; REASON='listen'
     sockfd = 7
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99009)
     backlog = 4096
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 8
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99011)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99011)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 9
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99012)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 9
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99012)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 9
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99013)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 9
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99013)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 7
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99015)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 7
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99015)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 8
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99016)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99016)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 8
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99017)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99017)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 7
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99020)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 7
     optname = 2
[40000] TRACE at socketwrappers.cpp:141 in bind; REASON='bind'
     sockfd = 7
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99020)
[40000] TRACE at socketwrappers.cpp:159 in listen; REASON='listen'
     sockfd = 7
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99020)
     backlog = 1024
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 8
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99022)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99022)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 9
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99023)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 9
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99023)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 9
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99024)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 9
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99024)
[40000] TRACE at miscwrappers.cpp:120 in pipe; REASON='promoting pipe() to socketpair()'
[40000] TRACE at socketwrappers.cpp:271 in socketpair; REASON='socketpair()'
     sv[0] = 8
     sv[1] = 9
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99027)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at coordinatorapi.cpp:623 in createNewConnectionBeforeFork; REASON='Got virtual pid from coordinator'
     hello_remote.virtualPid = 41000
[40000] TRACE at processinfo.cpp:357 in insertChild; REASON='Creating new virtualPid -> realPid mapping.'
     pid = 41000
     uniquePid = 6db90f3d5a9dd200-41000-546a1de8
[40000] TRACE at execwrappers.cpp:205 in fork; REASON='fork()ed [PARENT] done'
     child = 6db90f3d5a9dd200-41000-546a1de8
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6374
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 8
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99029)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 8
     optname = 2
[40000] TRACE at socketwrappers.cpp:141 in bind; REASON='bind'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99029)
[40000] TRACE at socketwrappers.cpp:159 in listen; REASON='listen'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99029)
     backlog = 1024
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6375
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40006
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6374
     th->virtual_tid = 40006
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40006
[41000] TRACE at virtualidtable.h:182 in printMaps; REASON='Virtual To Real Mappings:'
     _idMapTable.size() = 5
     out.str() = Pid Maps
      Virtual  ->  Real
	4213	->   4213
	40000	->   6365
	40002	->   6368
	40004	->   6369
	41000	->   6373

[41000] TRACE at virtualidtable.h:182 in printMaps; REASON='Virtual To Real Mappings:'
     _idMapTable.size() = 5
     out.str() = Pid Maps
      Virtual  ->  Real
	4213	->   4213
	40000	->   6365
	40002	->   6368
	40004	->   6369
	41000	->   6373

[41000] TRACE at uniquepid.cpp:199 in resetOnFork; REASON='Explicitly setting process UniquePid'
     newId = 6db90f3d5a9dd200-41000-546a1de8
[41000] TRACE at execwrappers.cpp:157 in pthread_atfork_child; REASON='fork()ed [CHILD]'
     child = 6db90f3d5a9dd200-41000-546a1de8
     parent = 6db90f3d5a9dd200-40000-546a1de8
[41000] TRACE at coordinatorapi.cpp:186 in resetOnFork; REASON='Informing coordinator of new process'
     UniquePid::ThisProcess() = 6db90f3d5a9dd200-41000-546a1de8
[41000] TRACE at virtualidtable.h:182 in printMaps; REASON='Virtual To Real Mappings:'
     _idMapTable.size() = 5
     out.str() = Pid Maps
      Virtual  ->  Real
	4213	->   4213
	40000	->   6365
	40002	->   6368
	40004	->   6369
	41000	->   6373

[41000] TRACE at virtualidtable.h:182 in printMaps; REASON='Virtual To Real Mappings:'
     _idMapTable.size() = 5
     out.str() = Pid Maps
      Virtual  ->  Real
	4213	->   4213
	40000	->   6365
	40002	->   6368
	40004	->   6369
	41000	->   6373

[41000] TRACE at dmtcpworker.cpp:349 in cleanupWorker; REASON='disconnecting from dmtcp coordinator'
[41000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6369
[41000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6368
[41000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6365
[41000] TRACE at execwrappers.cpp:199 in fork; REASON='fork() done [CHILD]'
     child = 6db90f3d5a9dd200-41000-546a1de8
     parent = 6db90f3d5a9dd200-40000-546a1de8
[41000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6373
     th->virtual_tid = 41000
[41000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[41000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6376
[40000] TRACE at miscwrappers.cpp:120 in pipe; REASON='promoting pipe() to socketpair()'
[40000] TRACE at socketwrappers.cpp:271 in socketpair; REASON='socketpair()'
     sv[0] = 10
     sv[1] = 11
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99030)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 12
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99032)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 12
     optname = 2
[40000] TRACE at socketwrappers.cpp:141 in bind; REASON='bind'
     sockfd = 12
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99032)
[40000] TRACE at socketwrappers.cpp:159 in listen; REASON='listen'
     sockfd = 12
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99032)
     backlog = 1024
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6377
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40008
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6375
     th->virtual_tid = 40008
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40008
[40000] TRACE at miscwrappers.cpp:120 in pipe; REASON='promoting pipe() to socketpair()'
[40000] TRACE at socketwrappers.cpp:271 in socketpair; REASON='socketpair()'
     sv[0] = 13
     sv[1] = 14
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99033)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at miscwrappers.cpp:120 in pipe; REASON='promoting pipe() to socketpair()'
[40000] TRACE at socketwrappers.cpp:271 in socketpair; REASON='socketpair()'
     sv[0] = 15
     sv[1] = 16
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99036)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 17
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99038)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 17
     optname = 2
[40000] TRACE at socketwrappers.cpp:141 in bind; REASON='bind'
     sockfd = 17
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99038)
[40000] TRACE at socketwrappers.cpp:159 in listen; REASON='listen'
     sockfd = 17
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99038)
     backlog = 1024
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6378
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40010
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6377
     th->virtual_tid = 40010
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40010
[41000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 41002
[41000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6376
     th->virtual_tid = 41002
[41000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 41002
[41000] TRACE at threadlist.cpp:318 in checkpointhread; REASON='after sigsetjmp/getcontext'
     curThread->tid = 6376
     curThread->virtual_tid = 41002
     curThread->saved_sp = 0x7fa89e55d790
[41000] TRACE at threadlist.cpp:334 in checkpointhread; REASON='before callbackSleepBetweenCheckpoint(0)'
[41000] TRACE at dmtcpworker.cpp:468 in waitForStage1Suspend; REASON='running'
[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for SUSPEND message'
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6379
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40012
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6378
     th->virtual_tid = 40012
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40012
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6380
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40014
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6379
     th->virtual_tid = 40014
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40014
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 18
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99040)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40016
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6380
     th->virtual_tid = 40016
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 18
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99040)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99041)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99041)
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40016
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99043)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99043)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99044)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99044)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99045)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99045)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
[40000] TRACE at socketwrappers.cpp:184 in process_accept; REASON='accepted incoming connection'
     sockfd = 17
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99046)
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 18
     optname = 18
[40000] TRACE at socketwrappers.cpp:184 in process_accept; REASON='accepted incoming connection'
     sockfd = 12
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99047)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 21
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99048)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 21
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99048)
[40000] TRACE at socketwrappers.cpp:184 in process_accept; REASON='accepted incoming connection'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99049)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99050)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99050)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
[40000] TRACE at socketwrappers.cpp:184 in process_accept; REASON='accepted incoming connection'
     sockfd = 17
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99051)
[40000] TRACE at socketwrappers.cpp:237 in setsockopt; REASON='setsockopt'
     ret = 0
     sockfd = 19
     optname = 18
[40000] TRACE at socketwrappers.cpp:184 in process_accept; REASON='accepted incoming connection'
     sockfd = 12
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99052)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 21
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99053)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 21
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99053)
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6381
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6382
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40018
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6381
     th->virtual_tid = 40018
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6380
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6380
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6379
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6379
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40018
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99054)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40020
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6382
     th->virtual_tid = 40020
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40020
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 21
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99054)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 22
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99056)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 22
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99056)
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99055)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 21
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99055)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 22
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99057)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 22
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99057)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 22
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99058)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 22
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99058)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 19
[40000] TRACE at socketwrappers.cpp:184 in process_accept; REASON='accepted incoming connection'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99059)
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6383
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6384
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40022
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6383
     th->virtual_tid = 40022
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6381
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6381
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40022
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 22
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99060)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40024
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6384
     th->virtual_tid = 40024
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40024
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 22
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99060)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 23
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99061)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 24
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99062)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 24
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99062)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 23
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99061)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 21
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99063)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 21
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99063)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 23
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99064)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 23
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99064)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 23
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99065)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 23
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99065)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 20
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 22
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99066)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 22
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99066)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
[40000] TRACE at socketwrappers.cpp:184 in process_accept; REASON='accepted incoming connection'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99067)
[40000] TRACE at socketwrappers.cpp:184 in process_accept; REASON='accepted incoming connection'
     sockfd = 8
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99068)
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6385
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6386
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40026
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6385
     th->virtual_tid = 40026
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6384
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6384
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6383
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6383
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6382
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6382
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40026
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 21
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99069)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40028
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6386
     th->virtual_tid = 40028
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40028
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 22
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99070)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 21
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99069)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 23
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99071)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 23
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99071)
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 22
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99070)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 23
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99072)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 23
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99072)
Process 0 of 2 is on slurm-compute1
iteration 0 on process 0
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 23
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99073)
     domain = 1
     type = 1
     protocol = 0
Process 1 of 2 is on slurm-compute2
iteration 0 on process 1
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 23
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99073)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 20
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 21
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99074)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 21
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99074)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
iteration 1 on process 0
iteration 1 on process 1
iteration 2 on process 1
iteration 2 on process 0
iteration 3 on process 1
iteration 3 on process 0
iteration 4 on process 1
iteration 4 on process 0
iteration 5 on process 1
iteration 5 on process 0
[40000] TRACE at dmtcpworker.cpp:474 in waitForStage1Suspend; REASON='got SUSPEND message, preparing to acquire all ThreadSync locks'
[40000] TRACE at threadsync.cpp:133 in acquireLocks; REASON='waiting for dmtcp_lock(): to get synchronized with _runCoordinatorCmd if we use
 DMTCP API'
[40000] TRACE at threadsync.cpp:136 in acquireLocks; REASON='Waiting for lock(&theCkptCanStart)'
[40000] TRACE at threadsync.cpp:139 in acquireLocks; REASON='Waiting for libdlLock'
[40000] TRACE at threadsync.cpp:142 in acquireLocks; REASON='Waiting for threads creation lock'
[40000] TRACE at threadsync.cpp:147 in acquireLocks; REASON='Waiting for other threads to exit DMTCP-Wrappers'
[40000] TRACE at threadsync.cpp:152 in acquireLocks; REASON='Waiting for newly created threads to finish initialization'
     _uninitializedThreadCount = 0
[40000] TRACE at threadsync.cpp:157 in acquireLocks; REASON='Done acquiring all locks'
[40000] TRACE at dmtcpworker.cpp:477 in waitForStage1Suspend; REASON='Starting checkpoint, suspending...'
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6386
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6385
[41000] TRACE at dmtcpworker.cpp:474 in waitForStage1Suspend; REASON='got SUSPEND message, preparing to acquire all ThreadSync locks'
[41000] TRACE at threadsync.cpp:133 in acquireLocks; REASON='waiting for dmtcp_lock(): to get synchronized with _runCoordinatorCmd if we use
 DMTCP API'
[41000] TRACE at threadsync.cpp:136 in acquireLocks; REASON='Waiting for lock(&theCkptCanStart)'
[41000] TRACE at threadsync.cpp:139 in acquireLocks; REASON='Waiting for libdlLock'
[41000] TRACE at threadsync.cpp:142 in acquireLocks; REASON='Waiting for threads creation lock'
[41000] TRACE at threadsync.cpp:147 in acquireLocks; REASON='Waiting for other threads to exit DMTCP-Wrappers'
[41000] TRACE at threadsync.cpp:152 in acquireLocks; REASON='Waiting for newly created threads to finish initialization'
     _uninitializedThreadCount = 0
[41000] TRACE at threadsync.cpp:157 in acquireLocks; REASON='Done acquiring all locks'
[41000] TRACE at dmtcpworker.cpp:477 in waitForStage1Suspend; REASON='Starting checkpoint, suspending...'
[40000] TRACE at threadlist.cpp:502 in stopthisthread; REASON='Thread after sigsetjmp/getcontext'
     curThread->tid = 6377
     curThread->virtual_tid = 40010
     curThread->saved_sp = 0x7fa89c429580
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[40000] TRACE at threadlist.cpp:530 in stopthisthread; REASON='User thread suspended'
     curThread->tid = 6377
[40000] TRACE at threadlist.cpp:502 in stopthisthread; REASON='Thread after sigsetjmp/getcontext'
     curThread->tid = 6375
     curThread->virtual_tid = 40008
     curThread->saved_sp = 0x7fa89ce2a540
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[40000] TRACE at threadlist.cpp:530 in stopthisthread; REASON='User thread suspended'
     curThread->tid = 6375
[40000] TRACE at threadlist.cpp:502 in stopthisthread; REASON='Thread after sigsetjmp/getcontext'
     curThread->tid = 6374
     curThread->virtual_tid = 40006
     curThread->saved_sp = 0x7fa89cf2b300
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[40000] TRACE at threadlist.cpp:530 in stopthisthread; REASON='User thread suspended'
     curThread->tid = 6374
[40000] TRACE at threadlist.cpp:502 in stopthisthread; REASON='Thread after sigsetjmp/getcontext'
     curThread->tid = 6369
     curThread->virtual_tid = 40004
     curThread->saved_sp = 0x7fa8a151f540
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[40000] TRACE at threadlist.cpp:530 in stopthisthread; REASON='User thread suspended'
     curThread->tid = 6369
[40000] TRACE at threadlist.cpp:502 in stopthisthread; REASON='Thread after sigsetjmp/getcontext'
     curThread->tid = 6365
     curThread->virtual_tid = 40000
     curThread->saved_sp = 0x7fffe5c57000
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[40000] TRACE at threadlist.cpp:530 in stopthisthread; REASON='User thread suspended'
     curThread->tid = 6365
[41000] TRACE at threadlist.cpp:502 in stopthisthread; REASON='Thread after sigsetjmp/getcontext'
     curThread->tid = 6373
     curThread->virtual_tid = 41000
     curThread->saved_sp = 0x7fffe5c57080
     __builtin_return_address(0) = 0x7fa8a00ee710
[41000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[41000] TRACE at threadlist.cpp:530 in stopthisthread; REASON='User thread suspended'
     curThread->tid = 6373
[41000] TRACE at threadlist.cpp:443 in suspendThreads; REASON='everything suspended'
     numUserThreads = 1
[41000] TRACE at siginfo.cpp:110 in saveSigHandlers; REASON='saving signal handlers'
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 18
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 15
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 14
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 13
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 12
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 10
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 3
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 2
[41000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 1
[41000] TRACE at threadlist.cpp:347 in checkpointhread; REASON='before callbackSleepBetweenCheckpoint(0)'
[41000] TRACE at dmtcpworker.cpp:483 in waitForStage2Checkpoint; REASON='suspended'
[41000] TRACE at threadsync.cpp:164 in releaseLocks; REASON='Releasing ThreadSync locks'
[41000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[40000] TRACE at threadlist.cpp:502 in stopthisthread; REASON='Thread after sigsetjmp/getcontext'
     curThread->tid = 6378
     curThread->virtual_tid = 40012
     curThread->saved_sp = 0x7fa89c3284c0
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[40000] TRACE at threadlist.cpp:530 in stopthisthread; REASON='User thread suspended'
     curThread->tid = 6378
[40000] TRACE at threadlist.cpp:443 in suspendThreads; REASON='everything suspended'
     numUserThreads = 6
[40000] TRACE at siginfo.cpp:110 in saveSigHandlers; REASON='saving signal handlers'
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 21
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 18
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 15
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 14
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 13
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 12
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 10
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 3
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 2
[40000] TRACE at siginfo.cpp:119 in saveSigHandlers; REASON='saving signal handler (non-default) for'
     sig = 1
[40000] TRACE at threadlist.cpp:347 in checkpointhread; REASON='before callbackSleepBetweenCheckpoint(0)'
[40000] TRACE at dmtcpworker.cpp:483 in waitForStage2Checkpoint; REASON='suspended'
[40000] TRACE at threadsync.cpp:164 in releaseLocks; REASON='Releasing ThreadSync locks'
[40000] TRACE at procname.cpp:64 in prctlGetProcessName; REASON='prctl(PR_GET_NAME, ...) succeeded'
     prgName.str = DMTCP:srun
[40000] TRACE at connectionlist.cpp:244 in list; REASON='ConnectionList'
     (dmtcp_get_uniquepid_str ? dmtcp_get_uniquepid_str() : __null) = 6db90f3d5a9dd200-40000-546a1de8
     o.str() =
0,1,2	6db90f3d5a9dd200-40000-546a1de8(99000)	:/dev/pts/1

[40000] TRACE at connectionlist.cpp:244 in list; REASON='ConnectionList'
     (dmtcp_get_uniquepid_str ? dmtcp_get_uniquepid_str() : __null) = 6db90f3d5a9dd200-40000-546a1de8
     o.str() =
3	6db90f3d5a9dd200-40000-546a1de8(99005)	<TCP Socket>
4	6db90f3d5a9dd200-40000-546a1de8(99006)	<TCP Socket>
5	6db90f3d5a9dd200-40000-546a1de8(99007)	<TCP Socket>
6	6db90f3d5a9dd200-40000-546a1de8(99008)	<TCP Socket>
7	6db90f3d5a9dd200-40000-546a1de8(99020)	<TCP Socket>
9	6db90f3d5a9dd200-40000-546a1de8(99028)	<TCP Socket>
8	6db90f3d5a9dd200-40000-546a1de8(99029)	<TCP Socket>
10	6db90f3d5a9dd200-40000-546a1de8(99030)	<TCP Socket>
11	6db90f3d5a9dd200-40000-546a1de8(99031)	<TCP Socket>
12	6db90f3d5a9dd200-40000-546a1de8(99032)	<TCP Socket>
13	6db90f3d5a9dd200-40000-546a1de8(99033)	<TCP Socket>
14	6db90f3d5a9dd200-40000-546a1de8(99034)	<TCP Socket>
15	6db90f3d5a9dd200-40000-546a1de8(99036)	<TCP Socket>
16	6db90f3d5a9dd200-40000-546a1de8(99037)	<TCP Socket>
17	6db90f3d5a9dd200-40000-546a1de8(99038)	<TCP Socket>
18	6db90f3d5a9dd200-40000-546a1de8(99046)	<TCP Socket>
19	6db90f3d5a9dd200-40000-546a1de8(99051)	<TCP Socket>

[40000] TRACE at connectionlist.cpp:244 in list; REASON='ConnectionList'
     (dmtcp_get_uniquepid_str ? dmtcp_get_uniquepid_str() : __null) = 6db90f3d5a9dd200-40000-546a1de8
     o.str() =

[41000] TRACE at connectionlist.cpp:244 in list; REASON='ConnectionList'
     (dmtcp_get_uniquepid_str ? dmtcp_get_uniquepid_str() : __null) = 6db90f3d5a9dd200-41000-546a1de8
     o.str() =
0,1,2	6db90f3d5a9dd200-40000-546a1de8(99000)	:/dev/pts/1

[41000] TRACE at connectionlist.cpp:244 in list; REASON='ConnectionList'
     (dmtcp_get_uniquepid_str ? dmtcp_get_uniquepid_str() : __null) = 6db90f3d5a9dd200-41000-546a1de8
     o.str() =
3	6db90f3d5a9dd200-40000-546a1de8(99005)	<TCP Socket>
4	6db90f3d5a9dd200-40000-546a1de8(99006)	<TCP Socket>
5	6db90f3d5a9dd200-40000-546a1de8(99007)	<TCP Socket>
6	6db90f3d5a9dd200-40000-546a1de8(99008)	<TCP Socket>
7	6db90f3d5a9dd200-40000-546a1de8(99020)	<TCP Socket>
8	6db90f3d5a9dd200-40000-546a1de8(99027)	<TCP Socket>

[41000] TRACE at connectionlist.cpp:244 in list; REASON='ConnectionList'
     (dmtcp_get_uniquepid_str ? dmtcp_get_uniquepid_str() : __null) = 6db90f3d5a9dd200-41000-546a1de8
     o.str() =

[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for FD_LEADER_ELECTION message'
[41000] TRACE at dmtcpworker.cpp:449 in waitForCoordinatorMsg; REASON='Computation information'
     msg.compGroup = 6db90f3d5a9dd200-40000-546a1de8
     msg.numPeers = 2
[41000] TRACE at connectionlist.cpp:88 in eventHook; REASON='locking...'
[41000] TRACE at connectionlist.cpp:90 in eventHook; REASON='locked'
[41000] TRACE at connectionlist.cpp:88 in eventHook; REASON='locking...'
[41000] TRACE at connectionlist.cpp:90 in eventHook; REASON='locked'
[41000] TRACE at connectionlist.cpp:88 in eventHook; REASON='locking...'
[41000] TRACE at connectionlist.cpp:90 in eventHook; REASON='locked'
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for FD_LEADER_ELECTION message'
[40000] TRACE at dmtcpworker.cpp:449 in waitForCoordinatorMsg; REASON='Computation information'
     msg.compGroup = 6db90f3d5a9dd200-40000-546a1de8
     msg.numPeers = 2
[40000] TRACE at connectionlist.cpp:88 in eventHook; REASON='locking...'
[40000] TRACE at connectionlist.cpp:90 in eventHook; REASON='locked'
[40000] TRACE at connectionlist.cpp:88 in eventHook; REASON='locking...'
[40000] TRACE at connectionlist.cpp:90 in eventHook; REASON='locked'
[40000] TRACE at connectionlist.cpp:88 in eventHook; REASON='locking...'
[40000] TRACE at connectionlist.cpp:90 in eventHook; REASON='locked'
[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for DRAIN message'
[41000] TRACE at pidwrappers.cpp:156 in tcgetpgrp; REASON='tcgetpgrp return value'
     fd = 9
     retval = 40000
[41000] TRACE at processinfo.cpp:481 in refresh; REASON='CHECK GROUP PID'
     _gid = 40000
     _fgid = 40000
     _ppid = 40000
     _pid = 41000
[41000] TRACE at connectionlist.cpp:94 in eventHook; REASON='draining...'
[41000] TRACE at connectionlist.cpp:96 in eventHook; REASON='drained'
[41000] TRACE at connectionlist.cpp:94 in eventHook; REASON='draining...'
[41000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 8
     _id = 6db90f3d5a9dd200-40000-546a1de8(99027)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(-1)
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for DRAIN message'
[40000] TRACE at pidwrappers.cpp:156 in tcgetpgrp; REASON='tcgetpgrp return value'
     fd = 20
     retval = 40000
[40000] TRACE at processinfo.cpp:481 in refresh; REASON='CHECK GROUP PID'
     _gid = 40000
     _fgid = 40000
     _ppid = 4213
     _pid = 40000
[40000] TRACE at connectionlist.cpp:94 in eventHook; REASON='draining...'
[40000] TRACE at pidwrappers.cpp:156 in tcgetpgrp; REASON='tcgetpgrp return value'
     fd = 0
     retval = 40000
[40000] TRACE at connectionlist.cpp:96 in eventHook; REASON='drained'
[40000] TRACE at connectionlist.cpp:94 in eventHook; REASON='draining...'
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 5
     _id = 6db90f3d5a9dd200-40000-546a1de8(99007)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(-1)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 6
     _id = 6db90f3d5a9dd200-40000-546a1de8(99008)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99007)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 9
     _id = 6db90f3d5a9dd200-40000-546a1de8(99028)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99027)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 10
     _id = 6db90f3d5a9dd200-40000-546a1de8(99030)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(-1)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 11
     _id = 6db90f3d5a9dd200-40000-546a1de8(99031)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99030)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 13
     _id = 6db90f3d5a9dd200-40000-546a1de8(99033)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(-1)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 14
     _id = 6db90f3d5a9dd200-40000-546a1de8(99034)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99033)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 15
     _id = 6db90f3d5a9dd200-40000-546a1de8(99036)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(-1)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 16
     _id = 6db90f3d5a9dd200-40000-546a1de8(99037)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99036)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 18
     _id = 6db90f3d5a9dd200-40000-546a1de8(99046)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(-1)
[40000] TRACE at socketconnection.cpp:366 in drain; REASON='Will drain socket'
     _hasLock = 1
     _fds[0] = 19
     _id = 6db90f3d5a9dd200-40000-546a1de8(99051)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(-1)
[41000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 8
     buffer.size() = 0
     (_dataSockets.size()) = 1
[41000] TRACE at connectionlist.cpp:96 in eventHook; REASON='drained'
[41000] TRACE at connectionlist.cpp:94 in eventHook; REASON='draining...'
[41000] TRACE at connectionlist.cpp:96 in eventHook; REASON='drained'
[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for CHECKPOINT message'
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 5
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 6
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 9
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 10
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 11
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 13
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 14
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 15
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] TRACE at kernelbufferdrainer.cpp:105 in onTimeoutInterval; REASON='buffer drain complete'
     _dataSockets[i]->socket().sockfd() = 16
     buffer.size() = 0
     (_dataSockets.size()) = 11
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 195
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 195
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 302
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 302
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 446
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 446
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 554
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 554
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 698
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 698
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 806
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 806
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 914
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 914
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 1058
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 1058
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 1166
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 1166
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 1310
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 1310
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 1418
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 1418
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 1526
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 1526
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 19
     buffer.size() = 1670
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] WARNING at kernelbufferdrainer.cpp:124 in onTimeoutInterval; REASON='JWARNING(false) failed'
     _dataSockets[i]->socket().sockfd() = 18
     buffer.size() = 1670
     WARN_INTERVAL_SEC = 10
Message: Still draining socket... perhaps remote host is not running under DMTCP?
[40000] TRACE at jsocket.cpp:735 in monitorSockets; REASON='accepting new connection'
     i = 4
     sk.sockfd() = 20
     _listenSockets[i].sockfd() = 12
     (strerror((*__errno_location ()))) = Success
[40000] WARNING at kernelbufferdrainer.cpp:61 in onConnect; REASON='JWARNING(false) failed'
     sock.sockfd() = 20
Message: we don't yet support checkpointing non-accepted connections... restore will likely fail.. closing connection
[40000] TRACE at jsocket.cpp:735 in monitorSockets; REASON='accepting new connection'
     i = 4
     sk.sockfd() = 20
     _listenSockets[i].sockfd() = 12
     (strerror((*__errno_location ()))) = Success
[40000] WARNING at kernelbufferdrainer.cpp:61 in onConnect; REASON='JWARNING(false) failed'
     sock.sockfd() = 20
Message: we don't yet support checkpointing non-accepted connections... restore will likely fail.. closing connection
[40000] TRACE at kernelbufferdrainer.cpp:84 in onDisconnect; REASON='found disconnected socket... marking it dead'
     fd = 19
     _reverseLookup[fd] = 6db90f3d5a9dd200-40000-546a1de8(99051)
     (strerror((*__errno_location ()))) = Success
[40000] TRACE at kernelbufferdrainer.cpp:84 in onDisconnect; REASON='found disconnected socket... marking it dead'
     fd = 18
     _reverseLookup[fd] = 6db90f3d5a9dd200-40000-546a1de8(99046)
     (strerror((*__errno_location ()))) = Success
[40000] TRACE at socketconnlist.cpp:58 in drain; REASON='recreating disconnected socket'
     id = 6db90f3d5a9dd200-40000-546a1de8(99046)
[40000] TRACE at socketconnection.cpp:335 in onError; REASON='Error.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99046)
[40000] TRACE at socketconnection.cpp:337 in onError; REASON='Creating dead socket.'
     _fds[0] = 18
     _fds.size() = 1
[40000] TRACE at socketconnlist.cpp:58 in drain; REASON='recreating disconnected socket'
     id = 6db90f3d5a9dd200-40000-546a1de8(99051)
[40000] TRACE at socketconnection.cpp:335 in onError; REASON='Error.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99051)
[40000] TRACE at socketconnection.cpp:337 in onError; REASON='Creating dead socket.'
     _fds[0] = 19
     _fds.size() = 1
[40000] TRACE at connectionlist.cpp:96 in eventHook; REASON='drained'
[40000] TRACE at connectionlist.cpp:94 in eventHook; REASON='draining...'
[40000] TRACE at connectionlist.cpp:96 in eventHook; REASON='drained'
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for CHECKPOINT message'
[41000] TRACE at dmtcpworker.cpp:508 in waitForStage2Checkpoint; REASON='got checkpoint message'
[41000] TRACE at connectionlist.cpp:100 in eventHook; REASON='preCKpt...'
[41000] TRACE at connectionlist.cpp:102 in eventHook; REASON='done preCkpt'
[41000] TRACE at connectionlist.cpp:100 in eventHook; REASON='preCKpt...'
[41000] TRACE at socketconnlist.cpp:71 in preCkpt; REASON='beginning handshakes'
[41000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99027)
     _fds[0] = 8
[40000] TRACE at dmtcpworker.cpp:508 in waitForStage2Checkpoint; REASON='got checkpoint message'
[40000] TRACE at connectionlist.cpp:100 in eventHook; REASON='preCKpt...'
[40000] TRACE at connectionlist.cpp:102 in eventHook; REASON='done preCkpt'
[40000] TRACE at connectionlist.cpp:100 in eventHook; REASON='preCKpt...'
[40000] TRACE at socketconnlist.cpp:71 in preCkpt; REASON='beginning handshakes'
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99007)
     _fds[0] = 5
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99008)
     _fds[0] = 6
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99028)
     _fds[0] = 9
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99030)
     _fds[0] = 10
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99031)
     _fds[0] = 11
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99033)
     _fds[0] = 13
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99034)
     _fds[0] = 14
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99036)
     _fds[0] = 15
[40000] TRACE at socketconnection.cpp:389 in doSendHandshakes; REASON='Sending handshake ...'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99037)
     _fds[0] = 16
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99007)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99008)
     _fds[0] = 5
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99008)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99007)
     _fds[0] = 6
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99028)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99027)
     _fds[0] = 9
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99030)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99031)
     _fds[0] = 10
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99031)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99030)
     _fds[0] = 11
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99033)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99034)
     _fds[0] = 13
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99034)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99033)
     _fds[0] = 14
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99036)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99037)
     _fds[0] = 15
[40000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99037)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99036)
     _fds[0] = 16
[40000] TRACE at socketconnlist.cpp:89 in preCkpt; REASON='handshaking done'
[40000] TRACE at connectionlist.cpp:102 in eventHook; REASON='done preCkpt'
[40000] TRACE at connectionlist.cpp:100 in eventHook; REASON='preCKpt...'
[40000] TRACE at connectionlist.cpp:102 in eventHook; REASON='done preCkpt'
[40000] TRACE at ckptserializer.cpp:545 in writeCkptImage; REASON='Thread performing checkpoint.'
     gettid() = 40002
[40000] TRACE at ckptserializer.cpp:210 in open_ckpt_to_write_gz; REASON='open_ckpt_to_write_gz
'
[40000] TRACE at processinfo.cpp:511 in serialize; REASON='Serialized process information'
     _sid = 1564
     _ppid = 4213
     _gid = 40000
     _fgid = 40000
     _procname = srun
     _hostname = slurm-master
     _launchCWD = /home/slurm
     _ckptCWD = /home/slurm
     _upid = 6db90f3d5a9dd200-40000-546a1de8
     _uppid = 0-0-0
     _compGroup = 6db90f3d5a9dd200-40000-546a1de8
     _numPeers = 2
     _noCoordinator = 0
     _argvSize = 21
     _envSize = 2884
     _elfType = 1
[41000] TRACE at socketconnection.cpp:404 in doRecvHandshakes; REASON='Received handshake.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99027)
     _remotePeerId = 6db90f3d5a9dd200-40000-546a1de8(99028)
     _fds[0] = 8
[41000] TRACE at socketconnlist.cpp:89 in preCkpt; REASON='handshaking done'
[41000] TRACE at connectionlist.cpp:102 in eventHook; REASON='done preCkpt'
[41000] TRACE at connectionlist.cpp:100 in eventHook; REASON='preCKpt...'
[41000] TRACE at connectionlist.cpp:102 in eventHook; REASON='done preCkpt'
[41000] TRACE at ckptserializer.cpp:545 in writeCkptImage; REASON='Thread performing checkpoint.'
     gettid() = 41002
[41000] TRACE at ckptserializer.cpp:210 in open_ckpt_to_write_gz; REASON='open_ckpt_to_write_gz
'
[41000] TRACE at processinfo.cpp:511 in serialize; REASON='Serialized process information'
     _sid = 1564
     _ppid = 40000
     _gid = 40000
     _fgid = 40000
     _procname = srun
     _hostname = slurm-master
     _launchCWD = /home/slurm
     _ckptCWD = /home/slurm
     _upid = 6db90f3d5a9dd200-41000-546a1de8
     _uppid = 6db90f3d5a9dd200-40000-546a1de8
     _compGroup = 6db90f3d5a9dd200-40000-546a1de8
     _numPeers = 2
     _noCoordinator = 0
     _argvSize = 21
     _envSize = 2884
     _elfType = 1
[41000] TRACE at processinfo.cpp:522 in serialize; REASON='Serializing ChildPid Table'
     _childTable.size() = 0
     o.filename() =
[41000] TRACE at ckptserializer.cpp:571 in writeCkptImage; REASON='MTCP is about to write checkpoint image.'
     ckptFilename = /home/slurm/ckpt_srun_6db90f3d5a9dd200-41000-546a1de8.dmtcp
[41000] TRACE at writeckpt.cpp:70 in mtcp_writememoryareas; REASON='Performing checkpoint.'
[40000] TRACE at processinfo.cpp:519 in serialize; REASON='This process is Root of Process Tree'
[40000] TRACE at processinfo.cpp:522 in serialize; REASON='Serializing ChildPid Table'
     _childTable.size() = 1
     o.filename() =
[40000] TRACE at ckptserializer.cpp:571 in writeCkptImage; REASON='MTCP is about to write checkpoint image.'
     ckptFilename = /home/slurm/ckpt_srun_6db90f3d5a9dd200-40000-546a1de8.dmtcp
[40000] TRACE at writeckpt.cpp:70 in mtcp_writememoryareas; REASON='Performing checkpoint.'
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x400000
     area->size = 1318912
     area->name = /usr/local/bin/srun
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x400000
     area->size = 1318912
     area->name = /usr/local/bin/srun
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x742000
     area->size = 24576
     area->name = /usr/local/bin/srun
     area->offset = 1318912
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x742000
     area->size = 24576
     area->name = /usr/local/bin/srun
     area->offset = 1318912
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x748000
     area->size = 20480
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x748000
     area->size = 20480
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x1e88000
     area->size = 745472
     area->name = [heap]
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x1e88000
     area->size = 4116480
     area->name = [heap]
     area->offset = 0
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa898000000
     area->size = 135168
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa898021000
     area->size = 66973696
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89cf2e000
     area->size = 32768
     area->name = /usr/lib64/libmunge.so.2.0.0
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d136000
     area->size = 4096
     area->name = /usr/lib64/libmunge.so.2.0.0
     area->offset = 32768
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d137000
     area->size = 12288
     area->name = /usr/local/lib/slurm/auth_munge.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d339000
     area->size = 4096
     area->name = /usr/local/lib/slurm/auth_munge.so
     area->offset = 8192
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d33a000
     area->size = 77824
     area->name = /usr/local/lib/slurm/mpi_pmi2.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d54c000
     area->size = 8192
     area->name = /usr/local/lib/slurm/mpi_pmi2.so
     area->offset = 73728
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d54e000
     area->size = 20480
     area->name = /usr/local/lib/slurm/launch_slurm.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d752000
     area->size = 4096
     area->name = /usr/local/lib/slurm/launch_slurm.so
     area->offset = 16384
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d753000
     area->size = 8192
     area->name = /usr/local/lib/slurm/switch_none.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d954000
     area->size = 4096
     area->name = /usr/local/lib/slurm/switch_none.so
     area->offset = 4096
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d955000
     area->size = 45056
     area->name = /usr/local/lib/slurm/select_linear.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89db5f000
     area->size = 4096
     area->name = /usr/local/lib/slurm/select_linear.so
     area->offset = 40960
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89db60000
     area->size = 4096
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89db61000
     area->size = 10485760
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa87c000000
     area->size = 135168
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89e793000
     area->size = 4096
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f194000
     area->size = 4096
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f195000
     area->size = 49152
     area->name = /lib64/libnss_files-2.12.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa87c021000
     area->size = 66973696
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f3a1000
     area->size = 4096
     area->name = /lib64/libnss_files-2.12.so
     area->offset = 49152
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f3a2000
     area->size = 4096
     area->name = /lib64/libnss_files-2.12.so
     area->offset = 53248
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f3a3000
     area->size = 28672
     area->name = /lib64/librt-2.12.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f5a9000
     area->size = 4096
     area->name = /lib64/librt-2.12.so
     area->offset = 24576
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f5aa000
     area->size = 4096
     area->name = /lib64/librt-2.12.so
     area->offset = 28672
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f5ab000
     area->size = 90112
     area->name = /lib64/libgcc_s-4.4.7-20120601.so.1
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f7c0000
     area->size = 4096
     area->name = /lib64/libgcc_s-4.4.7-20120601.so.1
     area->offset = 86016
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f7c1000
     area->size = 536576
     area->name = /lib64/libm-2.12.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa880000000
     area->size = 135168
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa880021000
     area->size = 66973696
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fa43000
     area->size = 4096
     area->name = /lib64/libm-2.12.so
     area->offset = 532480
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fa44000
     area->size = 4096
     area->name = /lib64/libm-2.12.so
     area->offset = 536576
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fa45000
     area->size = 950272
     area->name = /usr/lib64/libstdc++.so.6.0.13
     area->offset = 0
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa887eff000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa887f00000
     area->size = 1048576
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa888000000
     area->size = 135168
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa888021000
     area->size = 66973696
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fd2d000
     area->size = 28672
     area->name = /usr/lib64/libstdc++.so.6.0.13
     area->offset = 950272
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fd34000
     area->size = 8192
     area->name = /usr/lib64/libstdc++.so.6.0.13
     area->offset = 978944
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fd36000
     area->size = 86016
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fd4b000
     area->size = 1613824
     area->name = /lib64/libc-2.12.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa88c000000
     area->size = 3149824
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa88c301000
     area->size = 63959040
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa890000000
     area->size = 3149824
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa890301000
     area->size = 63959040
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa894000000
     area->size = 135168
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa894021000
     area->size = 66973696
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a00d5000
     area->size = 16384
     area->name = /lib64/libc-2.12.so
     area->offset = 1613824
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a00d9000
     area->size = 4096
     area->name = /lib64/libc-2.12.so
     area->offset = 1630208
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a00da000
     area->size = 20480
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a00df000
     area->size = 94208
     area->name = /lib64/libpthread-2.12.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a02f6000
     area->size = 4096
     area->name = /lib64/libpthread-2.12.so
     area->offset = 94208
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a02f7000
     area->size = 4096
     area->name = /lib64/libpthread-2.12.so
     area->offset = 98304
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a02f8000
     area->size = 16384
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a02fc000
     area->size = 8192
     area->name = /lib64/libdl-2.12.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a04fe000
     area->size = 4096
     area->name = /lib64/libdl-2.12.so
     area->offset = 8192
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a04ff000
     area->size = 4096
     area->name = /lib64/libdl-2.12.so
     area->offset = 12288
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0500000
     area->size = 155648
     area->name = /usr/local/lib/dmtcp/libdmtcp_pid.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0725000
     area->size = 8192
     area->name = /usr/local/lib/dmtcp/libdmtcp_pid.so
     area->offset = 151552
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0727000
     area->size = 823296
     area->name = /usr/local/lib/dmtcp/libdmtcp.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa898000000
     area->size = 135168
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa898021000
     area->size = 66973696
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a09ef000
     area->size = 20480
     area->name = /usr/local/lib/dmtcp/libdmtcp.so
     area->offset = 819200
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a09f4000
     area->size = 20480
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a09f9000
     area->size = 1105920
     area->name = /usr/local/lib/dmtcp/libdmtcp_ipc.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c028000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c029000
     area->size = 1048576
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c129000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c12a000
     area->size = 1048576
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c22a000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c22b000
     area->size = 1048576
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c32b000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c32c000
     area->size = 1048576
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0d07000
     area->size = 28672
     area->name = /usr/local/lib/dmtcp/libdmtcp_ipc.so
     area->offset = 1105920
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0d0e000
     area->size = 4096
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0d0f000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_dl.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0f0f000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_dl.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0f10000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_alloc.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1111000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_alloc.so
     area->offset = 4096
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1112000
     area->size = 77824
     area->name = /usr/local/lib/dmtcp/libdmtcp_batch-queue.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c42c000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89c42d000
     area->size = 10485760
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1325000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_batch-queue.so
     area->offset = 77824
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1326000
     area->size = 131072
     area->name = /lib64/ld-2.12.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1421000
     area->size = 4096
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1422000
     area->size = 1159168
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89ce2d000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89ce2e000
     area->size = 1048576
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89cf2e000
     area->size = 32768
     area->name = /usr/lib64/libmunge.so.2.0.0
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d136000
     area->size = 4096
     area->name = /usr/lib64/libmunge.so.2.0.0
     area->offset = 32768
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d137000
     area->size = 12288
     area->name = /usr/local/lib/slurm/auth_munge.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d339000
     area->size = 4096
     area->name = /usr/local/lib/slurm/auth_munge.so
     area->offset = 8192
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d33a000
     area->size = 77824
     area->name = /usr/local/lib/slurm/mpi_pmi2.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a153e000
     area->size = 28672
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d54c000
     area->size = 8192
     area->name = /usr/local/lib/slurm/mpi_pmi2.so
     area->offset = 73728
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1545000
     area->size = 4096
     area->name = /lib64/ld-2.12.so
     area->offset = 126976
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1546000
     area->size = 4096
     area->name = /lib64/ld-2.12.so
     area->offset = 131072
[41000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1547000
     area->size = 4096
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fffe526a000
     area->size = 10420224
     area->name = [stack]
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d54e000
     area->size = 20480
     area->name = /usr/local/lib/slurm/launch_slurm.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d752000
     area->size = 4096
     area->name = /usr/local/lib/slurm/launch_slurm.so
     area->offset = 16384
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d753000
     area->size = 8192
     area->name = /usr/local/lib/slurm/switch_none.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d954000
     area->size = 4096
     area->name = /usr/local/lib/slurm/switch_none.so
     area->offset = 4096
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89d955000
     area->size = 45056
     area->name = /usr/local/lib/slurm/select_linear.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89db5f000
     area->size = 4096
     area->name = /usr/local/lib/slurm/select_linear.so
     area->offset = 40960
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89db60000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89db61000
     area->size = 10485760
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89e793000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f194000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f195000
     area->size = 49152
     area->name = /lib64/libnss_files-2.12.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f3a1000
     area->size = 4096
     area->name = /lib64/libnss_files-2.12.so
     area->offset = 49152
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f3a2000
     area->size = 4096
     area->name = /lib64/libnss_files-2.12.so
     area->offset = 53248
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f3a3000
     area->size = 28672
     area->name = /lib64/librt-2.12.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f5a9000
     area->size = 4096
     area->name = /lib64/librt-2.12.so
     area->offset = 24576
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f5aa000
     area->size = 4096
     area->name = /lib64/librt-2.12.so
     area->offset = 28672
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f5ab000
     area->size = 90112
     area->name = /lib64/libgcc_s-4.4.7-20120601.so.1
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f7c0000
     area->size = 4096
     area->name = /lib64/libgcc_s-4.4.7-20120601.so.1
     area->offset = 86016
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89f7c1000
     area->size = 536576
     area->name = /lib64/libm-2.12.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fa43000
     area->size = 4096
     area->name = /lib64/libm-2.12.so
     area->offset = 532480
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fa44000
     area->size = 4096
     area->name = /lib64/libm-2.12.so
     area->offset = 536576
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fa45000
     area->size = 950272
     area->name = /usr/lib64/libstdc++.so.6.0.13
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fd2d000
     area->size = 28672
     area->name = /usr/lib64/libstdc++.so.6.0.13
     area->offset = 950272
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fd34000
     area->size = 8192
     area->name = /usr/lib64/libstdc++.so.6.0.13
     area->offset = 978944
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fd36000
     area->size = 86016
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa89fd4b000
     area->size = 1613824
     area->name = /lib64/libc-2.12.so
     area->offset = 0
[41000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fffe5ca5000
     area->size = 4096
     area->name = [vdso]
     area->offset = 0
[41000] TRACE at shareddata.cpp:189 in initialize; REASON='Shared area mapped'
     sharedDataHeader = 0x7fa89e561000
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a00d5000
     area->size = 16384
     area->name = /lib64/libc-2.12.so
     area->offset = 1613824
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a00d9000
     area->size = 4096
     area->name = /lib64/libc-2.12.so
     area->offset = 1630208
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a00da000
     area->size = 20480
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a00df000
     area->size = 94208
     area->name = /lib64/libpthread-2.12.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a02f6000
     area->size = 4096
     area->name = /lib64/libpthread-2.12.so
     area->offset = 94208
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a02f7000
     area->size = 4096
     area->name = /lib64/libpthread-2.12.so
     area->offset = 98304
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a02f8000
     area->size = 16384
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a02fc000
     area->size = 8192
     area->name = /lib64/libdl-2.12.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a04fe000
     area->size = 4096
     area->name = /lib64/libdl-2.12.so
     area->offset = 8192
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a04ff000
     area->size = 4096
     area->name = /lib64/libdl-2.12.so
     area->offset = 12288
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0500000
     area->size = 155648
     area->name = /usr/local/lib/dmtcp/libdmtcp_pid.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0725000
     area->size = 8192
     area->name = /usr/local/lib/dmtcp/libdmtcp_pid.so
     area->offset = 151552
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0727000
     area->size = 823296
     area->name = /usr/local/lib/dmtcp/libdmtcp.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a09ef000
     area->size = 20480
     area->name = /usr/local/lib/dmtcp/libdmtcp.so
     area->offset = 819200
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a09f4000
     area->size = 20480
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a09f9000
     area->size = 1105920
     area->name = /usr/local/lib/dmtcp/libdmtcp_ipc.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0d07000
     area->size = 28672
     area->name = /usr/local/lib/dmtcp/libdmtcp_ipc.so
     area->offset = 1105920
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0d0e000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0d0f000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_dl.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0f0f000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_dl.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a0f10000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_alloc.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1111000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_alloc.so
     area->offset = 4096
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1112000
     area->size = 77824
     area->name = /usr/local/lib/dmtcp/libdmtcp_batch-queue.so
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1325000
     area->size = 4096
     area->name = /usr/local/lib/dmtcp/libdmtcp_batch-queue.so
     area->offset = 77824
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1326000
     area->size = 131072
     area->name = /lib64/ld-2.12.so
     area->offset = 0
[41000] TRACE at ckptserializer.cpp:599 in writeCkptImage; REASON='checkpoint complete'
[41000] TRACE at threadlist.cpp:357 in checkpointhread; REASON='before callbackPostCheckpoint(0, NULL)'
[41000] TRACE at coordinatorapi.cpp:672 in sendCkptFilename; REASON='recording filenames'
     ckptFilename = /home/slurm/ckpt_srun_6db90f3d5a9dd200-41000-546a1de8.dmtcp
     hostname = slurm-master
[41000] TRACE at dmtcpworker.cpp:519 in waitForStage3Refill; REASON='checkpointed'
[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for REGISTER_NAME_SERVICE_DATA message'
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1421000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1422000
     area->size = 1159168
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a153e000
     area->size = 28672
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1545000
     area->size = 4096
     area->name = /lib64/ld-2.12.so
     area->offset = 126976
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1546000
     area->size = 4096
     area->name = /lib64/ld-2.12.so
     area->offset = 131072
[40000] TRACE at writeckpt.cpp:425 in writememoryarea; REASON='save anonymous'
     addr = 0x7fa8a1547000
     area->size = 4096
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fffe526a000
     area->size = 10420224
     area->name = [stack]
     area->offset = 0
[40000] TRACE at writeckpt.cpp:427 in writememoryarea; REASON='save anonymous'
     addr = 0x7fffe5ca5000
     area->size = 4096
     area->name = [vdso]
     area->offset = 0
[40000] TRACE at shareddata.cpp:189 in initialize; REASON='Shared area mapped'
     sharedDataHeader = 0x7fa89e561000
[40000] TRACE at ckptserializer.cpp:599 in writeCkptImage; REASON='checkpoint complete'
[40000] TRACE at threadlist.cpp:357 in checkpointhread; REASON='before callbackPostCheckpoint(0, NULL)'
[40000] TRACE at coordinatorapi.cpp:672 in sendCkptFilename; REASON='recording filenames'
     ckptFilename = /home/slurm/ckpt_srun_6db90f3d5a9dd200-40000-546a1de8.dmtcp
     hostname = slurm-master
[40000] TRACE at dmtcpworker.cpp:519 in waitForStage3Refill; REASON='checkpointed'
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for REGISTER_NAME_SERVICE_DATA message'
[41000] TRACE at dmtcpworker.cpp:528 in waitForStage3Refill; REASON='Key Value Pairs registered with the coordinator'
[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for SEND_QUERIES message'
[40000] TRACE at dmtcpworker.cpp:528 in waitForStage3Refill; REASON='Key Value Pairs registered with the coordinator'
[41000] TRACE at dmtcpworker.cpp:533 in waitForStage3Refill; REASON='Queries sent to the coordinator'
[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for REFILL message'
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for SEND_QUERIES message'
[40000] TRACE at dmtcpworker.cpp:533 in waitForStage3Refill; REASON='Queries sent to the coordinator'
[41000] TRACE at kernelbufferdrainer.cpp:167 in refillAllSockets; REASON='refilling socket buffers'
     _drainedData.size() = 1
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for REFILL message'
[40000] TRACE at kernelbufferdrainer.cpp:167 in refillAllSockets; REASON='refilling socket buffers'
     _drainedData.size() = 9
[41000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[41000] TRACE at kernelbufferdrainer.cpp:210 in refillAllSockets; REASON='buffers refilled'
[41000] TRACE at dmtcpworker.cpp:545 in waitForStage4Resume; REASON='refilled'
[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for RESUME message'
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:199 in refillAllSockets; REASON='repeating buffer back to peer'
     size = 0
[40000] TRACE at kernelbufferdrainer.cpp:210 in refillAllSockets; REASON='buffers refilled'
[40000] TRACE at dmtcpworker.cpp:545 in waitForStage4Resume; REASON='refilled'
[41000] TRACE at dmtcpworker.cpp:548 in waitForStage4Resume; REASON='got resume message'
[41000] TRACE at threadlist.cpp:361 in checkpointhread; REASON='resuming everything'
[41000] TRACE at threadlist.cpp:363 in checkpointhread; REASON='everything resumed'
[41000] TRACE at threadlist.cpp:334 in checkpointhread; REASON='before callbackSleepBetweenCheckpoint(0)'
[41000] TRACE at threadlist.cpp:532 in stopthisthread; REASON='User thread resuming'
     curThread->tid = 6373
[41000] TRACE at threadlist.cpp:543 in stopthisthread; REASON='User thread returning to user code'
     curThread->tid = 6373
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for RESUME message'
[40000] TRACE at dmtcpworker.cpp:548 in waitForStage4Resume; REASON='got resume message'
[40000] TRACE at threadlist.cpp:361 in checkpointhread; REASON='resuming everything'
[40000] TRACE at threadlist.cpp:532 in stopthisthread; REASON='User thread resuming'
     curThread->tid = 6377
[40000] TRACE at threadlist.cpp:543 in stopthisthread; REASON='User thread returning to user code'
     curThread->tid = 6377
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at threadlist.cpp:532 in stopthisthread; REASON='User thread resuming'
     curThread->tid = 6375
[40000] TRACE at threadlist.cpp:543 in stopthisthread; REASON='User thread returning to user code'
     curThread->tid = 6375
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at threadlist.cpp:532 in stopthisthread; REASON='User thread resuming'
     curThread->tid = 6374
[40000] TRACE at threadlist.cpp:543 in stopthisthread; REASON='User thread returning to user code'
     curThread->tid = 6374
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at threadlist.cpp:532 in stopthisthread; REASON='User thread resuming'
     curThread->tid = 6369
[40000] TRACE at threadlist.cpp:543 in stopthisthread; REASON='User thread returning to user code'
     curThread->tid = 6369
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at threadlist.cpp:532 in stopthisthread; REASON='User thread resuming'
     curThread->tid = 6365
[40000] TRACE at threadlist.cpp:543 in stopthisthread; REASON='User thread returning to user code'
     curThread->tid = 6365
     __builtin_return_address(0) = 0x7fa8a00ee710
[40000] TRACE at threadlist.cpp:532 in stopthisthread; REASON='User thread resuming'
     curThread->tid = 6378
[40000] TRACE at threadlist.cpp:543 in stopthisthread; REASON='User thread returning to user code'
     curThread->tid = 6378
     __builtin_return_address(0) = 0x7fa8a00ee710
slurmstepd-slurm-compute1: Message length of 2071343164 exceeds maximum of 1024
iteration 6 on process 0
slurmstepd-slurm-compute2: Message length of 2071343164 exceeds maximum of 1024
iteration 6 on process 1
iteration 7 on process 0
iteration 7 on process 1
iteration 8 on process 0
iteration 8 on process 1
iteration 9 on process 0
iteration 9 on process 1
iteration 10 on process 0
iteration 10 on process 1
iteration 11 on process 0
iteration 11 on process 1
iteration 12 on process 0
iteration 12 on process 1
iteration 13 on process 0
iteration 13 on process 1
iteration 14 on process 0
iteration 14 on process 1
iteration 15 on process 0
iteration 15 on process 1
iteration 16 on process 0
iteration 16 on process 1
iteration 17 on process 0
iteration 17 on process 1
iteration 18 on process 0
iteration 18 on process 1
iteration 19 on process 0
iteration 19 on process 1
iteration 20 on process 0
iteration 20 on process 1
iteration 21 on process 0
iteration 21 on process 1
iteration 22 on process 0
iteration 22 on process 1
iteration 23 on process 0
iteration 23 on process 1
iteration 24 on process 0
iteration 24 on process 1
iteration 25 on process 0
iteration 25 on process 1
iteration 26 on process 0
iteration 26 on process 1
iteration 27 on process 0
iteration 27 on process 1
iteration 28 on process 0
iteration 28 on process 1
iteration 29 on process 0
slurmstepd-slurm-compute1: Failed to send MESSAGE_TASK_EXIT: Transport endpoint is not connected
iteration 29 on process 1
slurmstepd-slurm-compute2: Failed to send MESSAGE_TASK_EXIT: Transport endpoint is not connected
iteration 30 on process 0
iteration 30 on process 1
iteration 31 on process 0
iteration 31 on process 1
iteration 32 on process 0
iteration 32 on process 1
iteration 33 on process 0
iteration 33 on process 1
iteration 34 on process 0
iteration 34 on process 1
iteration 35 on process 0
iteration 35 on process 1
iteration 36 on process 0
iteration 36 on process 1
iteration 37 on process 0
iteration 37 on process 1
iteration 38 on process 0
iteration 38 on process 1
iteration 39 on process 0
iteration 39 on process 1
iteration 40 on process 0
iteration 40 on process 1
iteration 41 on process 0
iteration 41 on process 1
iteration 42 on process 0
iteration 42 on process 1
iteration 43 on process 0
iteration 43 on process 1
iteration 44 on process 0
iteration 44 on process 1
iteration 45 on process 0
iteration 45 on process 1
iteration 46 on process 0
iteration 46 on process 1
iteration 47 on process 0
iteration 47 on process 1
iteration 48 on process 0
iteration 48 on process 1
iteration 49 on process 0
iteration 49 on process 1
Goodbye world from process 0 of 2
Goodbye world from process 1 of 2
[40000] TRACE at threadlist.cpp:363 in checkpointhread; REASON='everything resumed'
[40000] TRACE at threadlist.cpp:334 in checkpointhread; REASON='before callbackSleepBetweenCheckpoint(0)'
[40000] TRACE at dmtcpworker.cpp:468 in waitForStage1Suspend; REASON='running'
[40000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for SUSPEND message'
[41000] TRACE at dmtcpworker.cpp:468 in waitForStage1Suspend; REASON='running'
[41000] TRACE at dmtcpworker.cpp:428 in waitForCoordinatorMsg; REASON='waiting for SUSPEND message'
^Csrun: interrupt (one more within 1 sec to abort)
srun: tasks 0-1: running
^Csrun: sending Ctrl-C to job 107.0
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6403
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6404
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40030
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6403
     th->virtual_tid = 40030
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40030
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 18
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99075)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40032
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6404
     th->virtual_tid = 40032
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40032
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99076)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 18
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99075)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99077)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99077)
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99076)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99078)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99078)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99079)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99079)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 1635198781
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 18
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99080)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 18
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99080)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 18
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99082)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 18
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99082)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99083)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99083)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99084)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99084)
srun: Job step aborted: Waiting up to 2 seconds for job step to finish.
^Csrun: interrupt (one more within 1 sec to abort)
srun: tasks 0-1: running
^Csrun: sending Ctrl-C to job 107.0
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6406
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6407
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40034
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6406
     th->virtual_tid = 40034
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6404
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6404
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6403
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6403
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40034
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 18
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99085)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40036
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6407
     th->virtual_tid = 40036
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40036
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99086)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 18
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99085)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99087)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99087)
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99086)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99088)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99088)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 20
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99089)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 20
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99089)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 40030
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 18
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99090)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 18
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99090)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
^Csrun: forcing job termination
^Csrun: job abort in progress
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 18
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99092)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 18
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99092)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99093)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99093)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99094)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99094)
^C^C^C^C^C^C^C^C^C^C^C^C^C^Csrun: error: Timed out waiting for job step to complete
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 18
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99096)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 18
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99096)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99097)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99097)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 19
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99098)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 19
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99098)
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
[40000] TRACE at pid_miscwrappers.cpp:158 in __clone; REASON='Calling libc:__clone'
[40000] TRACE at pid_miscwrappers.cpp:167 in __clone; REASON='New thread created'
     tid = 6410
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 0
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 0
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99100)
     domain = 2
     type = 1
     protocol = 6
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 0
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99100)
[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 4
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99101)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 4
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99101)
[40000] TRACE at pid_miscwrappers.cpp:118 in clone_start; REASON='Calling user function'
     virtualTid = 40038
[40000] TRACE at threadlist.cpp:223 in updateTid; REASON='starting thread'
     th->tid = 6410
     th->virtual_tid = 40038
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6407
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6407
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6406
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6406
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6378
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6378
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6377
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6377
[40000] TRACE at threadlist.cpp:762 in addToActiveList; REASON='Killing zombie thread'
     thread->tid = 6369
[40000] TRACE at threadlist.cpp:795 in threadIsDead; REASON='Putting thread on freelist'
     thread->tid = 6369
[40000] TRACE at threadwrappers.cpp:66 in clone_start; REASON='Calling user function'
     gettid() = 40038
^Csrun: interrupt (abort already in progress)
srun: tasks 0-1: running
^Csrun: job abort in progress
^C[40000] TRACE at socketwrappers.cpp:59 in socket; REASON='socket created'
     ret = 4
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketconnection.cpp:195 in TcpConnection; REASON='Creating TcpConnection.'
     id() = 6db90f3d5a9dd200-40000-546a1de8(99102)
     domain = 1
     type = 1
     protocol = 0
[40000] TRACE at socketwrappers.cpp:121 in connect; REASON='connected'
     sockfd = 4
     con->id() = 6db90f3d5a9dd200-40000-546a1de8(99102)
[41000] TRACE at coordinatorapi.cpp:57 in dmtcp_CoordinatorAPI_EventHook; REASON='exit() in progress, disconnecting from dmtcp coordinator'
[41000] TRACE at threadlist.cpp:236 in killCkpthread; REASON='Kill checkpinthread'
     ckptThread->tid = 6376
[40000] TRACE at threadwrappers.cpp:148 in pthread_start; REASON='Thread returned'
     virtualTid = 8
[41000] TRACE at dmtcpworker.cpp:349 in cleanupWorker; REASON='disconnecting from dmtcp coordinator'
[40000] TRACE at dmtcpworker.cpp:349 in cleanupWorker; REASON='disconnecting from dmtcp coordinator'