Full thread dump Java HotSpot(TM) 64-Bit Server VM (19.0-b09 mixed mode): "Attach Listener" daemon prio=10 tid=0x000000005fa39000 nid=0x6412 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "IPC Server handler 9 on 9001" daemon prio=10 tid=0x00002aaab4022000 nid=0x6155 waiting on condition [0x000000004325f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 8 on 9001" daemon prio=10 tid=0x00002aaab4020000 nid=0x6154 waiting on condition [0x000000004315e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 7 on 9001" daemon prio=10 tid=0x00002aaab401e000 nid=0x6153 waiting on condition [0x000000004305d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 6 on 9001" daemon prio=10 tid=0x00002aaab401c000 nid=0x6152 waiting on condition [0x0000000042f5c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 5 on 9001" daemon prio=10 tid=0x00002aaab401a000 nid=0x6151 waiting on condition [0x0000000042e5b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 4 on 9001" daemon prio=10 tid=0x00002aaab4018000 nid=0x6150 waiting on condition [0x0000000042d5a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 3 on 9001" daemon prio=10 tid=0x00002aaab4016000 nid=0x614f waiting on condition [0x0000000042c59000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 2 on 9001" daemon prio=10 tid=0x00002aaab4014000 nid=0x614e waiting on condition [0x0000000042b58000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 1 on 9001" daemon prio=10 tid=0x00002aaab4012000 nid=0x614d waiting on condition [0x0000000042a57000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server handler 0 on 9001" daemon prio=10 tid=0x00002aaab4010000 nid=0x614c waiting on condition [0x0000000042956000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea41920> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:941) "IPC Server listener on 9001" daemon prio=10 tid=0x00002aaab400e800 nid=0x614b runnable [0x0000000042855000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00000007aea41e60> (a sun.nio.ch.Util$2) - locked <0x00000007aea41e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007aea41c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) at org.apache.hadoop.ipc.Server$Listener.run(Server.java:320) "IPC Server Responder" daemon prio=10 tid=0x00002aaab400d000 nid=0x614a runnable [0x0000000042754000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00000007aea425e0> (a sun.nio.ch.Util$2) - locked <0x00000007aea425d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007aea423c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.hadoop.ipc.Server$Responder.run(Server.java:480) "expireLaunchingTasks" prio=10 tid=0x00002aaab400b800 nid=0x6148 waiting on condition [0x0000000042653000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.mapred.JobTracker$ExpireLaunchingTasks.run(JobTracker.java:253) at java.lang.Thread.run(Thread.java:662) "retireJobs" prio=10 tid=0x00002aaab400a800 nid=0x6147 waiting on condition [0x0000000042552000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.mapred.JobTracker$RetireJobs.run(JobTracker.java:411) at java.lang.Thread.run(Thread.java:662) "expireTrackers" prio=10 tid=0x00002aaab400a000 nid=0x6146 waiting on condition [0x000000004224f000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.mapred.JobTracker$ExpireTrackers.run(JobTracker.java:336) at java.lang.Thread.run(Thread.java:662) "jobInitManager" daemon prio=10 tid=0x00002aaab40d2800 nid=0x6145 in Object.wait() [0x0000000042451000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007aea40858> (a java.util.ArrayList) at java.lang.Object.wait(Object.java:485) at org.apache.hadoop.mapred.EagerTaskInitializationListener$JobInitManager.run(EagerTaskInitializationListener.java:55) - locked <0x00000007aea40858> (a java.util.ArrayList) at java.lang.Thread.run(Thread.java:662) "LeaseChecker" daemon prio=10 tid=0x00002aaab410d800 nid=0x6143 waiting on condition [0x0000000042350000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1106) at java.lang.Thread.run(Thread.java:662) "Thread-14" prio=10 tid=0x00002aaab40bf800 nid=0x6141 waiting on condition [0x000000004214e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007aea002c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987) at java.util.concurrent.DelayQueue.take(DelayQueue.java:160) at java.util.concurrent.DelayQueue.take(DelayQueue.java:39) at org.apache.hadoop.mapred.JobEndNotifier$1.run(JobEndNotifier.java:50) at java.lang.Thread.run(Thread.java:662) "IPC Client (47) connection to sjc1-hadoop0.sjc1.carrieriq.com/10.202.50.70:9000 from hadoop" daemon prio=10 tid=0x00002aaab40bd000 nid=0x6127 runnable [0x0000000041b50000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00000007aea00458> (a sun.nio.ch.Util$2) - locked <0x00000007aea00468> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007aea00410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) at java.io.FilterInputStream.read(FilterInputStream.java:116) at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:276) at java.io.BufferedInputStream.fill(BufferedInputStream.java:218) at java.io.BufferedInputStream.read(BufferedInputStream.java:237) - locked <0x00000007aea025c8> (a java.io.BufferedInputStream) at java.io.DataInputStream.readInt(DataInputStream.java:370) at org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:501) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:446) "Timer-0" daemon prio=10 tid=0x00002aaab40b9000 nid=0x6109 in Object.wait() [0x00000000408d8000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007aea02c58> (a java.util.TaskQueue) at java.util.TimerThread.mainLoop(Timer.java:509) - locked <0x00000007aea02c58> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:462) "2091197094@qtp0-1 - Acceptor0 SelectChannelConnector@0.0.0.0:50030" prio=10 tid=0x000000005f678000 nid=0x6041 runnable [0x0000000041a4f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) - locked <0x00000007aea56ec8> (a sun.nio.ch.Util$2) - locked <0x00000007aea56eb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007aea44710> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:429) at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:185) at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:707) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:522) "1224577418@qtp0-0" prio=10 tid=0x000000005f737000 nid=0x6040 in Object.wait() [0x000000004043d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007aea43d88> (a org.mortbay.thread.QueuedThreadPool$PoolThread) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:565) - locked <0x00000007aea43d88> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "RMI TCP Accept-0" daemon prio=10 tid=0x000000005f59e000 nid=0x600c runnable [0x0000000041312000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390) - locked <0x00000007aea87790> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:462) at java.net.ServerSocket.accept(ServerSocket.java:430) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:34) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341) at java.lang.Thread.run(Thread.java:662) "RMI TCP Accept-11002" daemon prio=10 tid=0x000000005f596800 nid=0x600b runnable [0x0000000041211000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390) - locked <0x00000007aea87c40> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:462) at java.net.ServerSocket.accept(ServerSocket.java:430) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341) at java.lang.Thread.run(Thread.java:662) "RMI TCP Accept-0" daemon prio=10 tid=0x000000005f586800 nid=0x600a runnable [0x00000000407d7000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390) - locked <0x00000007aea879e8> (a java.net.SocksSocketImpl) at java.net.ServerSocket.implAccept(ServerSocket.java:462) at java.net.ServerSocket.accept(ServerSocket.java:430) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:369) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:341) at java.lang.Thread.run(Thread.java:662) "Low Memory Detector" daemon prio=10 tid=0x000000005f10b000 nid=0x6007 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread1" daemon prio=10 tid=0x000000005f109000 nid=0x6006 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "CompilerThread0" daemon prio=10 tid=0x000000005f103800 nid=0x6005 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x000000005f101000 nid=0x6004 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x000000005f0dd000 nid=0x6003 in Object.wait() [0x000000004158e000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007aea80c80> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) - locked <0x00000007aea80c80> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) "Reference Handler" daemon prio=10 tid=0x000000005f0db000 nid=0x6002 in Object.wait() [0x000000004148d000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007aea000f8> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:485) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116) - locked <0x00000007aea000f8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x000000005f07d000 nid=0x5ffe in Object.wait() [0x0000000040209000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x00000007aea417c8> (a org.apache.hadoop.ipc.RPC$Server) at java.lang.Object.wait(Object.java:485) at org.apache.hadoop.ipc.Server.join(Server.java:1126) - locked <0x00000007aea417c8> (a org.apache.hadoop.ipc.RPC$Server) at org.apache.hadoop.mapred.JobTracker.offerService(JobTracker.java:1866) at org.apache.hadoop.mapred.JobTracker.main(JobTracker.java:3713)