Process Thread Dump: 73 active threads Thread 21812 (ResponseProcessor for block blk_-8550997426499413457_56263): State: RUNNABLE Blocked count: 11 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) java.io.DataInputStream.readFully(DataInputStream.java:178) java.io.DataInputStream.readLong(DataInputStream.java:399) org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120) org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956) Thread 21808 (DataStreamer for file /hbase/au3_iu/6f10b2fa1903cf95c066e49556f6192e/.tmp/fc6b741630c3452a84c283da1b354d30 block blk_-8550997426499413457_56263): State: TIMED_WAITING Blocked count: 7659 Waited count: 7662 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805) Thread 21805 (IPC Client (47) connection to hmaster101.mentacapital.local/10.10.10.156:8020 from hbase): State: TIMED_WAITING Blocked count: 25 Waited count: 26 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:680) org.apache.hadoop.ipc.Client$Connection.run(Client.java:723) Thread 21803 (160800141@qtp-941866449-3): State: RUNNABLE Blocked count: 1 Waited count: 2 Stack: sun.management.ThreadImpl.getThreadInfo0(Native Method) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147) sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123) org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.java:149) org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:697) javax.servlet.http.HttpServlet.service(HttpServlet.java:707) javax.servlet.http.HttpServlet.service(HttpServlet.java:820) org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221) org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:829) org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212) org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399) org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766) org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450) org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230) org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) org.mortbay.jetty.Server.handle(Server.java:326) org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) Thread 21786 (ResponseProcessor for block blk_-3607823211736240800_56260): State: RUNNABLE Blocked count: 2 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128) java.io.DataInputStream.readFully(DataInputStream.java:178) java.io.DataInputStream.readLong(DataInputStream.java:399) org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120) org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956) Thread 21785 (DataStreamer for file /hbase/.logs/hb105.mentacapital.local.,60020,1339653832110/hb105.mentacapital.local.%2C60020%2C1339653832110.1340233811459 block blk_-3607823211736240800_56260): State: TIMED_WAITING Blocked count: 7942 Waited count: 8034 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805) Thread 6885 (regionserver60020-splits-1339824099835): State: WAITING Blocked count: 127 Waited count: 209 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@da6426 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 4956 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-2): State: WAITING Blocked count: 75 Waited count: 133 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 4950 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-1): State: WAITING Blocked count: 100 Waited count: 180 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 4945 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-0): State: WAITING Blocked count: 100 Waited count: 175 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 2743 (regionserver60020-smallCompactions-1339717999356): State: WAITING Blocked count: 137870 Waited count: 5766 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3e6ee7e4 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:220) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 831 (regionserver60020-largeCompactions-1339663832452): State: RUNNABLE Blocked count: 496027 Waited count: 3044 Stack: org.apache.hadoop.hbase.regionserver.StoreFileScanner.peek(StoreFileScanner.java:121) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:96) org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:368) org.apache.hadoop.hbase.regionserver.Store.compactStore(Store.java:1528) org.apache.hadoop.hbase.regionserver.Store.compact(Store.java:988) org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1188) org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.run(CompactionRequest.java:187) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 202 (sendParams-1): State: TIMED_WAITING Blocked count: 7 Waited count: 31437 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 94 (PostOpenDeployTasks:69eaa8adf8dab4c1e28905ce6eb726f7-EventThread): State: WAITING Blocked count: 0 Waited count: 4 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59c87031 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) Thread 93 (PostOpenDeployTasks:69eaa8adf8dab4c1e28905ce6eb726f7-SendThread(hmaster101.mentacapital.local.:2181)): State: RUNNABLE Blocked count: 2 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) Thread 83 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-2): State: WAITING Blocked count: 612 Waited count: 860 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 82 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-1): State: WAITING Blocked count: 605 Waited count: 859 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 81 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-0): State: WAITING Blocked count: 581 Waited count: 825 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 69 (SplitLogWorker-hb105.mentacapital.local.,60020,1339653832110): State: WAITING Blocked count: 4 Waited count: 3 Waiting on java.lang.Object@2e39060b Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:485) org.apache.hadoop.hbase.regionserver.SplitLogWorker.taskLoop(SplitLogWorker.java:205) org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:165) java.lang.Thread.run(Thread.java:619) Thread 68 (PRI IPC Server handler 9 on 60020): State: WAITING Blocked count: 29 Waited count: 91 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 67 (PRI IPC Server handler 8 on 60020): State: WAITING Blocked count: 32 Waited count: 94 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 66 (PRI IPC Server handler 7 on 60020): State: WAITING Blocked count: 30 Waited count: 88 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 65 (PRI IPC Server handler 6 on 60020): State: WAITING Blocked count: 28 Waited count: 90 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 64 (PRI IPC Server handler 5 on 60020): State: WAITING Blocked count: 28 Waited count: 87 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 63 (PRI IPC Server handler 4 on 60020): State: WAITING Blocked count: 36 Waited count: 89 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 62 (PRI IPC Server handler 3 on 60020): State: WAITING Blocked count: 34 Waited count: 88 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 61 (PRI IPC Server handler 2 on 60020): State: WAITING Blocked count: 48 Waited count: 100 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 60 (PRI IPC Server handler 1 on 60020): State: WAITING Blocked count: 34 Waited count: 91 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 59 (PRI IPC Server handler 0 on 60020): State: WAITING Blocked count: 49 Waited count: 106 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 58 (IPC Server handler 9 on 60020): State: RUNNABLE Blocked count: 130980 Waited count: 947471 Stack: org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340) org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105) org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066) org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710) org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753) org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651) org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343) sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376) Thread 57 (IPC Server handler 8 on 60020): State: WAITING Blocked count: 117086 Waited count: 934181 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 56 (IPC Server handler 7 on 60020): State: WAITING Blocked count: 105514 Waited count: 922107 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 55 (IPC Server handler 6 on 60020): State: WAITING Blocked count: 102307 Waited count: 918723 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 54 (IPC Server handler 5 on 60020): State: WAITING Blocked count: 121432 Waited count: 938055 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 53 (IPC Server handler 4 on 60020): State: RUNNABLE Blocked count: 130846 Waited count: 946687 Stack: org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340) org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105) org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066) org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710) org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753) org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651) org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343) sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376) Thread 52 (IPC Server handler 3 on 60020): State: RUNNABLE Blocked count: 113158 Waited count: 930380 Stack: org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.blockSeek(HFileReaderV2.java:802) org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.loadBlockAndSeekToKey(HFileReaderV2.java:710) org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:461) org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:475) org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:226) org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:145) org.apache.hadoop.hbase.regionserver.StoreScanner.(StoreScanner.java:130) org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2001) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.(HRegion.java:3274) org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1604) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1596) org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1572) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4065) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4039) org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1944) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3346) sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) Thread 51 (IPC Server handler 2 on 60020): State: WAITING Blocked count: 113506 Waited count: 930486 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348) Thread 50 (IPC Server handler 1 on 60020): State: RUNNABLE Blocked count: 113476 Waited count: 928781 Stack: org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340) org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105) org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066) org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710) org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753) org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651) org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343) sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376) Thread 49 (IPC Server handler 0 on 60020): State: RUNNABLE Blocked count: 121487 Waited count: 936162 Stack: org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340) org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105) org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406) org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310) org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327) org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066) org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710) org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753) org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651) org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513) org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343) sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364) org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376) Thread 10 (IPC Server listener on 60020): State: RUNNABLE Blocked count: 13590 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:610) Thread 22 (IPC Server Responder): State: RUNNABLE Blocked count: 6533 Waited count: 6452 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) org.apache.hadoop.hbase.ipc.HBaseServer$Responder.doRunLoop(HBaseServer.java:799) org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:782) Thread 48 (Timer-0): State: TIMED_WAITING Blocked count: 1 Waited count: 19351 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:509) java.util.TimerThread.run(Timer.java:462) Thread 47 (1136341612@qtp-941866449-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030): State: RUNNABLE Blocked count: 1 Waited count: 1 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498) org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192) org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124) org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Thread 32 (regionserver60020.leaseChecker): State: TIMED_WAITING Blocked count: 0 Waited count: 4559366 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) java.util.concurrent.DelayQueue.poll(DelayQueue.java:201) org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:83) java.lang.Thread.run(Thread.java:619) Thread 31 (regionserver60020.compactionChecker): State: TIMED_WAITING Blocked count: 75 Waited count: 60 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91) org.apache.hadoop.hbase.Chore.run(Chore.java:75) java.lang.Thread.run(Thread.java:619) Thread 30 (regionserver60020.cacheFlusher): State: TIMED_WAITING Blocked count: 40438 Waited count: 70695 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) java.util.concurrent.DelayQueue.poll(DelayQueue.java:201) java.util.concurrent.DelayQueue.poll(DelayQueue.java:39) org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:218) java.lang.Thread.run(Thread.java:619) Thread 38 (regionserver60020.logRoller): State: TIMED_WAITING Blocked count: 1226 Waited count: 59826 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:76) java.lang.Thread.run(Thread.java:619) Thread 44 (Timer thread for monitoring jvm): State: TIMED_WAITING Blocked count: 4 Waited count: 58050 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:509) java.util.TimerThread.run(Timer.java:462) Thread 43 (Timer thread for monitoring hbase): State: TIMED_WAITING Blocked count: 3 Waited count: 58050 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:509) java.util.TimerThread.run(Timer.java:462) Thread 42 (regionserver60020.decayingSampleTick.1): State: TIMED_WAITING Blocked count: 0 Waited count: 1160870 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) java.util.concurrent.DelayQueue.take(DelayQueue.java:164) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 41 (regionserver60020.logSyncer): State: TIMED_WAITING Blocked count: 2698 Waited count: 582724 Stack: java.lang.Thread.sleep(Native Method) org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1193) java.lang.Thread.run(Thread.java:619) Thread 40 (LeaseChecker): State: TIMED_WAITING Blocked count: 19346 Waited count: 618946 Stack: java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1302) java.lang.Thread.run(Thread.java:619) Thread 37 (DestroyJavaVM): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: Thread 33 (IPC Client (47) connection to hmaster101.mentacapital.local./10.10.10.156:60000 from hbase): State: TIMED_WAITING Blocked count: 193043 Waited count: 193043 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:459) org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:504) Thread 29 (regionserver60020-EventThread): State: WAITING Blocked count: 4 Waited count: 3 Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3003e926 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:158) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493) Thread 28 (regionserver60020-SendThread(hmaster101.mentacapital.local.:2181)): State: RUNNABLE Blocked count: 240 Waited count: 0 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035) Thread 26 (regionserver60020): State: TIMED_WAITING Blocked count: 193160 Waited count: 386109 Stack: java.lang.Object.wait(Native Method) org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91) org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:55) org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:714) java.lang.Thread.run(Thread.java:619) Thread 25 (LRU Statistics #0): State: TIMED_WAITING Blocked count: 0 Waited count: 1942 Stack: sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963) java.util.concurrent.DelayQueue.take(DelayQueue.java:164) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907) java.lang.Thread.run(Thread.java:619) Thread 23 (main.LruBlockCache.EvictionThread): State: WAITING Blocked count: 29 Waited count: 30 Waiting on org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread@6267fe80 Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:485) org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:594) java.lang.Thread.run(Thread.java:619) Thread 21 (Timer thread for monitoring rpc): State: TIMED_WAITING Blocked count: 1 Waited count: 58051 Stack: java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:509) java.util.TimerThread.run(Timer.java:462) Thread 20 (IPC Reader 9 on port 60020): State: RUNNABLE Blocked count: 1371 Waited count: 1359 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 19 (IPC Reader 8 on port 60020): State: RUNNABLE Blocked count: 1391 Waited count: 1359 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 18 (IPC Reader 7 on port 60020): State: RUNNABLE Blocked count: 1371 Waited count: 1359 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 17 (IPC Reader 6 on port 60020): State: RUNNABLE Blocked count: 1367 Waited count: 1359 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 16 (IPC Reader 5 on port 60020): State: RUNNABLE Blocked count: 1359 Waited count: 1359 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 15 (IPC Reader 4 on port 60020): State: RUNNABLE Blocked count: 1361 Waited count: 1359 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 14 (IPC Reader 3 on port 60020): State: RUNNABLE Blocked count: 1362 Waited count: 1360 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 13 (IPC Reader 2 on port 60020): State: RUNNABLE Blocked count: 1361 Waited count: 1360 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 12 (IPC Reader 1 on port 60020): State: RUNNABLE Blocked count: 1362 Waited count: 1360 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 11 (IPC Reader 0 on port 60020): State: RUNNABLE Blocked count: 1368 Waited count: 1359 Stack: sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502) org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488) java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) java.lang.Thread.run(Thread.java:619) Thread 5 (Signal Dispatcher): State: RUNNABLE Blocked count: 0 Waited count: 0 Stack: Thread 3 (Finalizer): State: WAITING Blocked count: 5664 Waited count: 5626 Waiting on java.lang.ref.ReferenceQueue$Lock@7f81497d Stack: java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134) java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159) Thread 2 (Reference Handler): State: WAITING Blocked count: 7586 Waited count: 7517 Waiting on java.lang.ref.Reference$Lock@22a010ba Stack: java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:485) java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)