2015-06-15 11:19:38 Full thread dump Java HotSpot(TM) 64-Bit Server VM (24.55-b03 mixed mode): "ResponseProcessor for block BP-1564683557-Master-1431405724229:blk_1073745331_4526" daemon prio=10 tid=0x00007fee28051800 nid=0x5a59 runnable [0x00007fee13d29000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000798113020> (a sun.nio.ch.Util$2) - locked <0x0000000798113008> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007a6dd03e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:1998) at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:176) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:798) "DataStreamer for file /hbasedata/data/testing/test/7a7b21e639344e3759db94d4582ce449/.tmp/d490bbe617db47528fca676efe97d2c4 block BP-1564683557-Master-1431405724229:blk_1073745331_4526" daemon prio=10 tid=0x00007fee302d1800 nid=0x5a56 in Object.wait() [0x00007fee13a26000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:503) - locked <0x000000078023fbf8> (a java.util.LinkedList) "hconnection-0x66148e7b-shared--pool3-t6" daemon prio=10 tid=0x00007fee28417000 nid=0x5a54 waiting on condition [0x00007fee13b27000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000794b528f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "hconnection-0x66148e7b-shared--pool3-t5" daemon prio=10 tid=0x00007fee2833b800 nid=0x5a53 waiting on condition [0x00007fee13c28000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000794b528f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "IPC Client (209179201) connection to /R2:16020 from sas" daemon prio=10 tid=0x00007fee282b0000 nid=0x5a47 in Object.wait() [0x00007fee13e2a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:537) - locked <0x000000078023fcd0> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:552) "regionserver//R1:16020-splits-1434347377305" prio=10 tid=0x0000000000eb3000 nid=0x5a1d waiting on condition [0x00007fee14d3d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007934795e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "IPC Client (209179201) connection to /Master:9999 from sas" daemon prio=10 tid=0x00007fee2c09f800 nid=0x59e5 in Object.wait() [0x00007fee17060000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:904) - locked <0x00000007936cbb88> (a org.apache.hadoop.ipc.Client$Connection) at org.apache.hadoop.ipc.Client$Connection.run(Client.java:949) "ResponseProcessor for block BP-1564683557-Master-1431405724229:blk_1073745320_4515" daemon prio=10 tid=0x00007fee28404000 nid=0x576e runnable [0x00007fee1412d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000007a4154a00> (a sun.nio.ch.Util$2) - locked <0x00000007a41549e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007a6dd0940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:1998) at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:176) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:798) "DataStreamer for file /hbasedata/WALs/R1,16020,1432798738361/R1%2C16020%2C1432798738361.default.1434347311675 block BP-1564683557-Master-1431405724229:blk_1073745320_4515" daemon prio=10 tid=0x00007fee3011f000 nid=0x576a in Object.wait() [0x00007fee1422e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:503) - locked <0x00000007a78d2898> (a java.util.LinkedList) "1547712307@qtp-843966615-49" daemon prio=10 tid=0x00007fee2c0a1000 nid=0x5666 in Object.wait() [0x00007fee16656000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) - locked <0x0000000792ca18b8> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "2081346502@qtp-843966615-48" daemon prio=10 tid=0x00007fee2c0a0000 nid=0x5665 in Object.wait() [0x00007fee1402c000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) - locked <0x0000000796877fb8> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "910391358@qtp-843966615-45" daemon prio=10 tid=0x00007fee2c0fd800 nid=0x5662 in Object.wait() [0x00007fee16454000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) - locked <0x00000007968781e8> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "753426310@qtp-843966615-40" daemon prio=10 tid=0x00007fee2c0d0000 nid=0x7ff0 in Object.wait() [0x00007fee14531000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626) - locked <0x0000000796877ca8> (a org.mortbay.thread.QueuedThreadPool$PoolThread) "RS_CLOSE_REGION-R1:16020-0" prio=10 tid=0x00007fee282a3800 nid=0x57ab waiting on condition [0x00007fee14939000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000793783400> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "Attach Listener" daemon prio=10 tid=0x00007fee30ecb000 nid=0x34f1 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "RS_OPEN_REGION-R1:16020-2" prio=10 tid=0x00007fee306ce800 nid=0x3c8a waiting on condition [0x00007fee16252000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000793782f68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RS_OPEN_REGION-R1:16020-1" prio=10 tid=0x00007fee3013e000 nid=0x3c89 waiting on condition [0x00007fee14430000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000793782f68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "ReplicationExecutor-0" daemon prio=10 tid=0x00007fee280d2000 nid=0x3bf2 waiting on condition [0x00007fee1432f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007934fe468> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020-shortCompactions-1432798748986" daemon prio=10 tid=0x00007fee28070800 nid=0x3bd8 runnable [0x00007fee14837000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:192) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:379) - locked <0x00000007802991f0> (a java.lang.Object) at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.readChannelFully(PacketReceiver.java:258) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:209) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:171) at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:102) at org.apache.hadoop.hdfs.RemoteBlockReader2.readNextPacket(RemoteBlockReader2.java:186) at org.apache.hadoop.hdfs.RemoteBlockReader2.read(RemoteBlockReader2.java:146) - locked <0x00000007802971d0> (a org.apache.hadoop.hdfs.RemoteBlockReader2) at org.apache.hadoop.hdfs.DFSInputStream$ByteArrayStrategy.doRead(DFSInputStream.java:686) at org.apache.hadoop.hdfs.DFSInputStream.readBuffer(DFSInputStream.java:742) - eliminated <0x0000000780297028> (a org.apache.hadoop.hdfs.DFSInputStream) at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:799) at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:840) - locked <0x0000000780297028> (a org.apache.hadoop.hdfs.DFSInputStream) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.hbase.io.hfile.HFileBlock.readWithExtra(HFileBlock.java:678) at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1369) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1588) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1467) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:430) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.readNextDataBlock(HFileReaderV2.java:699) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.next(HFileReaderV2.java:819) at org.apache.hadoop.hbase.io.HalfStoreFileReader$1.next(HalfStoreFileReader.java:160) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.next(StoreFileScanner.java:139) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:107) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:539) at org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Compactor.java:245) at org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:102) at org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:110) at org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1167) at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1610) at org.apache.hadoop.hbase.regionserver.CompactSplitThread$CompactionRunner.run(CompactSplitThread.java:511) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RS_OPEN_REGION-R1:16020-0" prio=10 tid=0x0000000000f6b800 nid=0x3bb9 waiting on condition [0x00007fee14c3c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000793782f68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RS_LOG_REPLAY_OPS-R1:16020-1" prio=10 tid=0x00007fee310ea000 nid=0x3bab waiting on condition [0x00007fee14b3b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007934fe168> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "org.apache.hadoop.hdfs.PeerCache@6d79dea2" daemon prio=10 tid=0x00007fee2c4f6000 nid=0x3ba3 waiting on condition [0x00007fee14a3a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:244) at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:41) at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:119) at java.lang.Thread.run(Thread.java:745) "RS_LOG_REPLAY_OPS-R1:16020-0" prio=10 tid=0x00007fee310e8000 nid=0x3b9f waiting on condition [0x00007fee14e3e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007934fe168> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "R1,16020,1432798738361-HeapMemoryTunerChore" daemon prio=10 tid=0x00007fee310df800 nid=0x3b9e in Object.wait() [0x00007fee14f3f000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) - locked <0x0000000793804e38> (a java.lang.Object) at org.apache.hadoop.hbase.Chore.run(Chore.java:88) at java.lang.Thread.run(Thread.java:745) "SplitLogWorker-R1:16020" prio=10 tid=0x00007fee310dd000 nid=0x3b9d in Object.wait() [0x00007fee15040000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination.taskLoop(ZkSplitLogWorkerCoordination.java:444) - locked <0x00000007934dead0> (a java.lang.Object) at org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:142) at java.lang.Thread.run(Thread.java:745) "R1:16020Replication Statistics #0" daemon prio=10 tid=0x00007fee310db800 nid=0x3b9c waiting on condition [0x00007fee15141000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007934fe1b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020-EventThread" daemon prio=10 tid=0x00007fee310d9800 nid=0x3b9b waiting on condition [0x00007fee15242000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000793804e50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "regionserver//R1:16020-SendThread(ZK:2191)" daemon prio=10 tid=0x00007fee310be800 nid=0x3b9a runnable [0x00007fee15343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000793804e98> (a sun.nio.ch.Util$2) - locked <0x0000000793804eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000793811198> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "regionserver//R1:16020.leaseChecker" daemon prio=10 tid=0x00007fee310bb000 nid=0x3b99 waiting on condition [0x00007fee15444000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:94) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020.nonceCleaner" daemon prio=10 tid=0x00007fee310b9000 nid=0x3b98 in Object.wait() [0x00007fee15545000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) - locked <0x00000007934da0c0> (a java.lang.Object) at org.apache.hadoop.hbase.Chore.run(Chore.java:88) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020.periodicFlusher" daemon prio=10 tid=0x00007fee310b7000 nid=0x3b97 in Object.wait() [0x00007fee15646000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) - locked <0x000000079325c660> (a java.lang.Object) at org.apache.hadoop.hbase.Chore.run(Chore.java:88) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020.compactionChecker" daemon prio=10 tid=0x00007fee310b4800 nid=0x3b96 in Object.wait() [0x00007fee15747000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) - locked <0x00000007933f8d28> (a java.lang.Object) at org.apache.hadoop.hbase.Chore.run(Chore.java:88) at java.lang.Thread.run(Thread.java:745) "MemStoreFlusher.1" prio=10 tid=0x00007fee310b3000 nid=0x3b95 waiting on condition [0x00007fee15848000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007932c6dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:256) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:68) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:205) at java.lang.Thread.run(Thread.java:745) "MemStoreFlusher.0" prio=10 tid=0x00007fee310b1000 nid=0x3b94 waiting on condition [0x00007fee15949000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007932c6dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:256) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:68) at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:205) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020.logRoller" daemon prio=10 tid=0x00007fee310af800 nid=0x3b93 in Object.wait() [0x00007fee15a4a000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:116) - locked <0x00000007932f18d0> (a java.util.concurrent.atomic.AtomicBoolean) at java.lang.Thread.run(Thread.java:745) "HBase-Metrics2-1" daemon prio=10 tid=0x00007fee310a3800 nid=0x3b92 waiting on condition [0x00007fee15b4b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007934dc370> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "sync.4" prio=10 tid=0x00007fee310a4800 nid=0x3b91 waiting on condition [0x00007fee15c4c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007934dc3a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1282) at java.lang.Thread.run(Thread.java:745) "sync.3" prio=10 tid=0x00007fee3109f800 nid=0x3b90 waiting on condition [0x00007fee15d4d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007934dc3e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1282) at java.lang.Thread.run(Thread.java:745) "sync.2" prio=10 tid=0x00007fee3109e000 nid=0x3b8f waiting on condition [0x00007fee15e4e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079347c5c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1282) at java.lang.Thread.run(Thread.java:745) "sync.1" prio=10 tid=0x00007fee3109c000 nid=0x3b8e waiting on condition [0x00007fee15f4f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079347c610> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1282) at java.lang.Thread.run(Thread.java:745) "sync.0" prio=10 tid=0x00007fee31095000 nid=0x3b8d waiting on condition [0x00007fee16050000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079347c598> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1282) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020.append-pool1-t1" prio=10 tid=0x00007fee31093800 nid=0x3b8c waiting on condition [0x00007fee16151000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079347c520> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:45) at com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:55) at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:123) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "LeaseRenewer:sas@mycluster" daemon prio=10 tid=0x00007fee3105c000 nid=0x3b88 waiting on condition [0x00007fee16353000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:438) at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71) at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:298) at java.lang.Thread.run(Thread.java:745) "IPC Parameter Sending Thread #0" daemon prio=10 tid=0x00007fee3101b800 nid=0x3b84 waiting on condition [0x00007fee16555000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000793158b48> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "LruBlockCacheStatsExecutor" daemon prio=10 tid=0x00007fee3100d000 nid=0x3b82 waiting on condition [0x00007fee16757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000793460ad8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020.LruBlockCache.EvictionThread" daemon prio=10 tid=0x00007fee31007000 nid=0x3b81 in Object.wait() [0x00007fee16858000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:791) - locked <0x00000007934656c0> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread) at java.lang.Thread.run(Thread.java:745) "IPC Client (209179201) connection to /Master:16020 from sas" daemon prio=10 tid=0x00007fee30ff8000 nid=0x3b80 in Object.wait() [0x00007fee16959000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:537) - locked <0x0000000792d79640> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:552) "JvmPauseMonitor" daemon prio=10 tid=0x00007fee30f5c000 nid=0x3b7f waiting on condition [0x00007fee16a5a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:159) at java.lang.Thread.run(Thread.java:745) "regionserver//R1:16020-EventThread" daemon prio=10 tid=0x00007fee30f3a000 nid=0x3b7e waiting on condition [0x00007fee16b5b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x000000079325c6d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "regionserver//R1:16020-SendThread(ZK:2191)" daemon prio=10 tid=0x00007fee30f19800 nid=0x3b7d runnable [0x00007fee16c5c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x000000079325c6c0> (a sun.nio.ch.Util$2) - locked <0x000000079325c678> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007933055e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "regionserver//R1:16020" prio=10 tid=0x00007fee30ee8800 nid=0x3b7c in Object.wait() [0x00007fee16d5d000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92) - locked <0x0000000792b35fa0> (a java.lang.Object) at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:56) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:881) at java.lang.Thread.run(Thread.java:745) "Timer-0" daemon prio=10 tid=0x00007fee30dd9800 nid=0x3b7b in Object.wait() [0x00007fee16e5e000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x00000007932f18e8> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "1915826467@qtp-843966615-1 - Acceptor0 SelectChannelConnector@0.0.0.0:16030" daemon prio=10 tid=0x00007fee30df3800 nid=0x3b7a runnable [0x00007fee16f5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000007932c6de8> (a sun.nio.ch.Util$2) - locked <0x00000007932c6e00> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007931aa5d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) "ReplicationRpcServer.handler=2,queue=0,port=16020" daemon prio=10 tid=0x00007fee30da4000 nid=0x3b76 waiting on condition [0x00007fee17161000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e04a40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "ReplicationRpcServer.handler=1,queue=0,port=16020" daemon prio=10 tid=0x00007fee30da2000 nid=0x3b75 waiting on condition [0x00007fee17262000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e04a40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "ReplicationRpcServer.handler=0,queue=0,port=16020" daemon prio=10 tid=0x00007fee30da0000 nid=0x3b74 waiting on condition [0x00007fee17363000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e04a40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=9,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d9e000 nid=0x3b73 waiting on condition [0x00007fee17464000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e048d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=8,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d9b800 nid=0x3b72 waiting on condition [0x00007fee17565000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792df25e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=7,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d99800 nid=0x3b71 waiting on condition [0x00007fee17666000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e048d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=6,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d97800 nid=0x3b70 waiting on condition [0x00007fee17767000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792df25e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=5,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d95800 nid=0x3b6f waiting on condition [0x00007fee17868000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e048d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=4,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d93800 nid=0x3b6e waiting on condition [0x00007fee17969000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792df25e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=3,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d91000 nid=0x3b6d waiting on condition [0x00007fee17a6a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e048d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=2,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d8f000 nid=0x3b6c waiting on condition [0x00007fee17b6b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792df25e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=1,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d8d000 nid=0x3b6b waiting on condition [0x00007fee17c6c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e048d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "PriorityRpcServer.handler=0,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d8b000 nid=0x3b6a waiting on condition [0x00007fee17d6d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792df25e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=29,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d89000 nid=0x3b69 waiting on condition [0x00007fee17e6e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=28,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d87000 nid=0x3b68 waiting on condition [0x00007fee17f6f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=27,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d84800 nid=0x3b67 waiting on condition [0x00007fee18070000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=26,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d82800 nid=0x3b66 waiting on condition [0x00007fee18171000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=25,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d80800 nid=0x3b65 waiting on condition [0x00007fee18272000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=24,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d7e800 nid=0x3b64 waiting on condition [0x00007fee18373000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=23,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d7c800 nid=0x3b63 waiting on condition [0x00007fee18474000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=22,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d7a000 nid=0x3b62 waiting on condition [0x00007fee18575000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=21,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d78000 nid=0x3b61 waiting on condition [0x00007fee18676000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=20,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d76000 nid=0x3b60 waiting on condition [0x00007fee18777000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=19,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d74000 nid=0x3b5f waiting on condition [0x00007fee18878000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=18,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d71800 nid=0x3b5e waiting on condition [0x00007fee18979000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=17,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d6f800 nid=0x3b5d waiting on condition [0x00007fee18a7a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=16,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d6d800 nid=0x3b5c waiting on condition [0x00007fee18b7b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=15,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d6b800 nid=0x3b5b waiting on condition [0x00007fee18c7c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=14,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d69800 nid=0x3b5a waiting on condition [0x00007fee18d7d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=13,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d67800 nid=0x3b59 waiting on condition [0x00007fee18e7e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=12,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d65000 nid=0x3b58 waiting on condition [0x00007fee18f7f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=11,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d63000 nid=0x3b57 waiting on condition [0x00007fee19080000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=10,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d61000 nid=0x3b56 waiting on condition [0x00007fee19181000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=9,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d5f000 nid=0x3b55 waiting on condition [0x00007fee19282000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=8,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d5d000 nid=0x3b54 waiting on condition [0x00007fee19383000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=7,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d5b000 nid=0x3b53 waiting on condition [0x00007fee19484000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=6,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d58800 nid=0x3b52 waiting on condition [0x00007fee19585000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=5,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d56800 nid=0x3b51 waiting on condition [0x00007fee19686000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=4,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d54800 nid=0x3b50 waiting on condition [0x00007fee19787000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=3,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d52800 nid=0x3b4f waiting on condition [0x00007fee19888000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=2,queue=2,port=16020" daemon prio=10 tid=0x00007fee30d50000 nid=0x3b4e waiting on condition [0x00007fee19989000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=1,queue=1,port=16020" daemon prio=10 tid=0x00007fee30d4e000 nid=0x3b4d waiting on condition [0x00007fee19a8a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "B.defaultRpcServer.handler=0,queue=0,port=16020" daemon prio=10 tid=0x00007fee30d4a800 nid=0x3b4c waiting on condition [0x00007fee19b8b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000792e0c0e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.take(BoundedPriorityBlockingQueue.java:213) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) "RpcServer.listener,port=16020" daemon prio=10 tid=0x00007fee30d49000 nid=0x3b4b runnable [0x00007fee19c8c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792e00000> (a sun.nio.ch.Util$2) - locked <0x0000000792dfffe8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792dde4d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener.run(RpcServer.java:674) "RpcServer.responder" daemon prio=10 tid=0x00007fee30d47800 nid=0x3b4a runnable [0x00007fee19d8d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000007931f2e00> (a sun.nio.ch.Util$2) - locked <0x00000007931f2de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007932318d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.hadoop.hbase.ipc.RpcServer$Responder.doRunLoop(RpcServer.java:913) at org.apache.hadoop.hbase.ipc.RpcServer$Responder.run(RpcServer.java:860) "main-EventThread" daemon prio=10 tid=0x00007fee30d0f000 nid=0x3b49 waiting on condition [0x00007fee19e8e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007931b9f50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494) "main-SendThread(ZK:2191)" daemon prio=10 tid=0x00007fee30d11800 nid=0x3b48 runnable [0x00007fee19f8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x00000007932c7718> (a sun.nio.ch.Util$2) - locked <0x00000007932c7700> (a java.util.Collections$UnmodifiableSet) - locked <0x00000007931aa620> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081) "Thread-5" daemon prio=10 tid=0x00007fee30cb1000 nid=0x3b47 runnable [0x00007fee1a090000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method) at org.apache.hadoop.net.unix.DomainSocketWatcher.access$800(DomainSocketWatcher.java:52) at org.apache.hadoop.net.unix.DomainSocketWatcher$1.run(DomainSocketWatcher.java:457) at java.lang.Thread.run(Thread.java:745) "Timer for 'HBase' metrics system" daemon prio=10 tid=0x00007fee30aaf800 nid=0x3b44 in Object.wait() [0x00007fee1a7b6000] java.lang.Thread.State: TIMED_WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.util.TimerThread.mainLoop(Timer.java:552) - locked <0x0000000792fc5920> (a java.util.TaskQueue) at java.util.TimerThread.run(Timer.java:505) "RpcServer.reader=9,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee30a4b000 nid=0x3b43 runnable [0x00007fee1a8b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792e91ca8> (a sun.nio.ch.Util$2) - locked <0x0000000792e05fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792df4f08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792e06000> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=8,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee30a30000 nid=0x3b42 runnable [0x00007fee1a9b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792e05fb8> (a sun.nio.ch.Util$2) - locked <0x0000000792e05fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792dfa7d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792e05f88> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=7,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee30a15000 nid=0x3b41 runnable [0x00007fee1aab9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792e05fa0> (a sun.nio.ch.Util$2) - locked <0x0000000792e05f58> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792dde2d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792e05f70> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=6,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee309f9800 nid=0x3b40 runnable [0x00007fee1abba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792e05f28> (a sun.nio.ch.Util$2) - locked <0x0000000792e05f40> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792dde368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792e05ef8> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=5,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee309de800 nid=0x3b3f runnable [0x00007fee1acbb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792e05f10> (a sun.nio.ch.Util$2) - locked <0x0000000792e05ec8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792dde3f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792e05ee0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=4,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee30928800 nid=0x3b3e runnable [0x00007fee1adbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792e05e98> (a sun.nio.ch.Util$2) - locked <0x0000000792e05eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792dde488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792e05e80> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=3,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee30927000 nid=0x3b3c runnable [0x00007fee1aebd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792dcf400> (a sun.nio.ch.Util$2) - locked <0x0000000792dcf418> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792db46a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792dcf430> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=2,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee30925800 nid=0x3b3b runnable [0x00007fee1afbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792dcf448> (a sun.nio.ch.Util$2) - locked <0x0000000792dcf460> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792db4610> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792dcf388> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=1,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee30924800 nid=0x3b3a runnable [0x00007fee1b0bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792dcf3a0> (a sun.nio.ch.Util$2) - locked <0x0000000792dcf3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792db4580> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792dcf3d0> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "RpcServer.reader=0,bindAddress=R1,port=16020" daemon prio=10 tid=0x00007fee30924000 nid=0x3b39 runnable [0x00007fee1b3c8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87) - locked <0x0000000792dcf3e8> (a sun.nio.ch.Util$2) - locked <0x0000000792dcf310> (a java.util.Collections$UnmodifiableSet) - locked <0x0000000792db44f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:571) - locked <0x0000000792dcf328> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader) at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:558) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "Service Thread" daemon prio=10 tid=0x00007fee30116000 nid=0x3b30 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" daemon prio=10 tid=0x00007fee30113800 nid=0x3b2f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" daemon prio=10 tid=0x00007fee30111000 nid=0x3b2e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" daemon prio=10 tid=0x00007fee3010e800 nid=0x3b2d runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Surrogate Locker Thread (Concurrent GC)" daemon prio=10 tid=0x00007fee30104800 nid=0x3b2c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE "Finalizer" daemon prio=10 tid=0x00007fee300ed000 nid=0x3b2b in Object.wait() [0x00007fee2260e000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135) - locked <0x0000000792b84fc0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189) "Reference Handler" daemon prio=10 tid=0x00007fee300e9000 nid=0x3b2a in Object.wait() [0x00007fee2270f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:503) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133) - locked <0x0000000792b84fd8> (a java.lang.ref.Reference$Lock) "main" prio=10 tid=0x00007fee3000c000 nid=0x3b23 in Object.wait() [0x00007fee37e9f000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x0000000792b46ad8> (a java.lang.Thread) at java.lang.Thread.join(Thread.java:1281) - locked <0x0000000792b46ad8> (a java.lang.Thread) at java.lang.Thread.join(Thread.java:1355) at org.apache.hadoop.hbase.util.HasThread.join(HasThread.java:89) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:66) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:87) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126) at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2502) "VM Thread" prio=10 tid=0x00007fee300e6800 nid=0x3b29 runnable "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x00007fee3001d000 nid=0x3b24 runnable "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x00007fee3001f000 nid=0x3b25 runnable "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x00007fee30021000 nid=0x3b26 runnable "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x00007fee30023000 nid=0x3b27 runnable "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x00007fee300a3000 nid=0x3b28 runnable "VM Periodic Task Thread" prio=10 tid=0x00007fee30121000 nid=0x3b31 waiting on condition JNI global references: 456