Process Thread Dump:
73 active threads
Thread 21812 (ResponseProcessor for block blk_-8550997426499413457_56263):
State: RUNNABLE
Blocked count: 11
Waited count: 0
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
java.io.DataInputStream.readFully(DataInputStream.java:178)
java.io.DataInputStream.readLong(DataInputStream.java:399)
org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120)
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956)
Thread 21808 (DataStreamer for file /hbase/au3_iu/6f10b2fa1903cf95c066e49556f6192e/.tmp/fc6b741630c3452a84c283da1b354d30 block blk_-8550997426499413457_56263):
State: TIMED_WAITING
Blocked count: 7659
Waited count: 7662
Stack:
java.lang.Object.wait(Native Method)
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805)
Thread 21805 (IPC Client (47) connection to hmaster101.mentacapital.local/10.10.10.156:8020 from hbase):
State: TIMED_WAITING
Blocked count: 25
Waited count: 26
Stack:
java.lang.Object.wait(Native Method)
org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:680)
org.apache.hadoop.ipc.Client$Connection.run(Client.java:723)
Thread 21803 (160800141@qtp-941866449-3):
State: RUNNABLE
Blocked count: 1
Waited count: 2
Stack:
sun.management.ThreadImpl.getThreadInfo0(Native Method)
sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.java:149)
org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:697)
javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:829)
org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
org.mortbay.jetty.Server.handle(Server.java:326)
org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
Thread 21786 (ResponseProcessor for block blk_-3607823211736240800_56260):
State: RUNNABLE
Blocked count: 2
Waited count: 0
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
java.io.DataInputStream.readFully(DataInputStream.java:178)
java.io.DataInputStream.readLong(DataInputStream.java:399)
org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120)
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956)
Thread 21785 (DataStreamer for file /hbase/.logs/hb105.mentacapital.local.,60020,1339653832110/hb105.mentacapital.local.%2C60020%2C1339653832110.1340233811459 block blk_-3607823211736240800_56260):
State: TIMED_WAITING
Blocked count: 7942
Waited count: 8034
Stack:
java.lang.Object.wait(Native Method)
org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805)
Thread 6885 (regionserver60020-splits-1339824099835):
State: WAITING
Blocked count: 127
Waited count: 209
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@da6426
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 4956 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-2):
State: WAITING
Blocked count: 75
Waited count: 133
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 4950 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-1):
State: WAITING
Blocked count: 100
Waited count: 180
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 4945 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-0):
State: WAITING
Blocked count: 100
Waited count: 175
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 2743 (regionserver60020-smallCompactions-1339717999356):
State: WAITING
Blocked count: 137870
Waited count: 5766
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3e6ee7e4
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:220)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 831 (regionserver60020-largeCompactions-1339663832452):
State: RUNNABLE
Blocked count: 496027
Waited count: 3044
Stack:
org.apache.hadoop.hbase.regionserver.StoreFileScanner.peek(StoreFileScanner.java:121)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:96)
org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:368)
org.apache.hadoop.hbase.regionserver.Store.compactStore(Store.java:1528)
org.apache.hadoop.hbase.regionserver.Store.compact(Store.java:988)
org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1188)
org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.run(CompactionRequest.java:187)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 202 (sendParams-1):
State: TIMED_WAITING
Blocked count: 7
Waited count: 31437
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 94 (PostOpenDeployTasks:69eaa8adf8dab4c1e28905ce6eb726f7-EventThread):
State: WAITING
Blocked count: 0
Waited count: 4
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59c87031
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493)
Thread 93 (PostOpenDeployTasks:69eaa8adf8dab4c1e28905ce6eb726f7-SendThread(hmaster101.mentacapital.local.:2181)):
State: RUNNABLE
Blocked count: 2
Waited count: 0
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274)
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035)
Thread 83 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-2):
State: WAITING
Blocked count: 612
Waited count: 860
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 82 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-1):
State: WAITING
Blocked count: 605
Waited count: 859
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 81 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-0):
State: WAITING
Blocked count: 581
Waited count: 825
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 69 (SplitLogWorker-hb105.mentacapital.local.,60020,1339653832110):
State: WAITING
Blocked count: 4
Waited count: 3
Waiting on java.lang.Object@2e39060b
Stack:
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:485)
org.apache.hadoop.hbase.regionserver.SplitLogWorker.taskLoop(SplitLogWorker.java:205)
org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:165)
java.lang.Thread.run(Thread.java:619)
Thread 68 (PRI IPC Server handler 9 on 60020):
State: WAITING
Blocked count: 29
Waited count: 91
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 67 (PRI IPC Server handler 8 on 60020):
State: WAITING
Blocked count: 32
Waited count: 94
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 66 (PRI IPC Server handler 7 on 60020):
State: WAITING
Blocked count: 30
Waited count: 88
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 65 (PRI IPC Server handler 6 on 60020):
State: WAITING
Blocked count: 28
Waited count: 90
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 64 (PRI IPC Server handler 5 on 60020):
State: WAITING
Blocked count: 28
Waited count: 87
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 63 (PRI IPC Server handler 4 on 60020):
State: WAITING
Blocked count: 36
Waited count: 89
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 62 (PRI IPC Server handler 3 on 60020):
State: WAITING
Blocked count: 34
Waited count: 88
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 61 (PRI IPC Server handler 2 on 60020):
State: WAITING
Blocked count: 48
Waited count: 100
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 60 (PRI IPC Server handler 1 on 60020):
State: WAITING
Blocked count: 34
Waited count: 91
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 59 (PRI IPC Server handler 0 on 60020):
State: WAITING
Blocked count: 49
Waited count: 106
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 58 (IPC Server handler 9 on 60020):
State: RUNNABLE
Blocked count: 130980
Waited count: 947471
Stack:
org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
java.lang.reflect.Method.invoke(Method.java:597)
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
Thread 57 (IPC Server handler 8 on 60020):
State: WAITING
Blocked count: 117086
Waited count: 934181
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 56 (IPC Server handler 7 on 60020):
State: WAITING
Blocked count: 105514
Waited count: 922107
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 55 (IPC Server handler 6 on 60020):
State: WAITING
Blocked count: 102307
Waited count: 918723
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 54 (IPC Server handler 5 on 60020):
State: WAITING
Blocked count: 121432
Waited count: 938055
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 53 (IPC Server handler 4 on 60020):
State: RUNNABLE
Blocked count: 130846
Waited count: 946687
Stack:
org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
java.lang.reflect.Method.invoke(Method.java:597)
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
Thread 52 (IPC Server handler 3 on 60020):
State: RUNNABLE
Blocked count: 113158
Waited count: 930380
Stack:
org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.blockSeek(HFileReaderV2.java:802)
org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.loadBlockAndSeekToKey(HFileReaderV2.java:710)
org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:461)
org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:475)
org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:226)
org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:145)
org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:130)
org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2001)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3274)
org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1604)
org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1596)
org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1572)
org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4065)
org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4039)
org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1944)
org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3346)
sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
java.lang.reflect.Method.invoke(Method.java:597)
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
Thread 51 (IPC Server handler 2 on 60020):
State: WAITING
Blocked count: 113506
Waited count: 930486
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
Thread 50 (IPC Server handler 1 on 60020):
State: RUNNABLE
Blocked count: 113476
Waited count: 928781
Stack:
org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
java.lang.reflect.Method.invoke(Method.java:597)
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
Thread 49 (IPC Server handler 0 on 60020):
State: RUNNABLE
Blocked count: 121487
Waited count: 936162
Stack:
org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
java.lang.reflect.Method.invoke(Method.java:597)
org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
Thread 10 (IPC Server listener on 60020):
State: RUNNABLE
Blocked count: 13590
Waited count: 0
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:610)
Thread 22 (IPC Server Responder):
State: RUNNABLE
Blocked count: 6533
Waited count: 6452
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
org.apache.hadoop.hbase.ipc.HBaseServer$Responder.doRunLoop(HBaseServer.java:799)
org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:782)
Thread 48 (Timer-0):
State: TIMED_WAITING
Blocked count: 1
Waited count: 19351
Stack:
java.lang.Object.wait(Native Method)
java.util.TimerThread.mainLoop(Timer.java:509)
java.util.TimerThread.run(Timer.java:462)
Thread 47 (1136341612@qtp-941866449-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030):
State: RUNNABLE
Blocked count: 1
Waited count: 1
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Thread 32 (regionserver60020.leaseChecker):
State: TIMED_WAITING
Blocked count: 0
Waited count: 4559366
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:83)
java.lang.Thread.run(Thread.java:619)
Thread 31 (regionserver60020.compactionChecker):
State: TIMED_WAITING
Blocked count: 75
Waited count: 60
Stack:
java.lang.Object.wait(Native Method)
org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
org.apache.hadoop.hbase.Chore.run(Chore.java:75)
java.lang.Thread.run(Thread.java:619)
Thread 30 (regionserver60020.cacheFlusher):
State: TIMED_WAITING
Blocked count: 40438
Waited count: 70695
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
java.util.concurrent.DelayQueue.poll(DelayQueue.java:39)
org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:218)
java.lang.Thread.run(Thread.java:619)
Thread 38 (regionserver60020.logRoller):
State: TIMED_WAITING
Blocked count: 1226
Waited count: 59826
Stack:
java.lang.Object.wait(Native Method)
org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:76)
java.lang.Thread.run(Thread.java:619)
Thread 44 (Timer thread for monitoring jvm):
State: TIMED_WAITING
Blocked count: 4
Waited count: 58050
Stack:
java.lang.Object.wait(Native Method)
java.util.TimerThread.mainLoop(Timer.java:509)
java.util.TimerThread.run(Timer.java:462)
Thread 43 (Timer thread for monitoring hbase):
State: TIMED_WAITING
Blocked count: 3
Waited count: 58050
Stack:
java.lang.Object.wait(Native Method)
java.util.TimerThread.mainLoop(Timer.java:509)
java.util.TimerThread.run(Timer.java:462)
Thread 42 (regionserver60020.decayingSampleTick.1):
State: TIMED_WAITING
Blocked count: 0
Waited count: 1160870
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 41 (regionserver60020.logSyncer):
State: TIMED_WAITING
Blocked count: 2698
Waited count: 582724
Stack:
java.lang.Thread.sleep(Native Method)
org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1193)
java.lang.Thread.run(Thread.java:619)
Thread 40 (LeaseChecker):
State: TIMED_WAITING
Blocked count: 19346
Waited count: 618946
Stack:
java.lang.Thread.sleep(Native Method)
org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1302)
java.lang.Thread.run(Thread.java:619)
Thread 37 (DestroyJavaVM):
State: RUNNABLE
Blocked count: 0
Waited count: 0
Stack:
Thread 33 (IPC Client (47) connection to hmaster101.mentacapital.local./10.10.10.156:60000 from hbase):
State: TIMED_WAITING
Blocked count: 193043
Waited count: 193043
Stack:
java.lang.Object.wait(Native Method)
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:459)
org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:504)
Thread 29 (regionserver60020-EventThread):
State: WAITING
Blocked count: 4
Waited count: 3
Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3003e926
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493)
Thread 28 (regionserver60020-SendThread(hmaster101.mentacapital.local.:2181)):
State: RUNNABLE
Blocked count: 240
Waited count: 0
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274)
org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035)
Thread 26 (regionserver60020):
State: TIMED_WAITING
Blocked count: 193160
Waited count: 386109
Stack:
java.lang.Object.wait(Native Method)
org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:55)
org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:714)
java.lang.Thread.run(Thread.java:619)
Thread 25 (LRU Statistics #0):
State: TIMED_WAITING
Blocked count: 0
Waited count: 1942
Stack:
sun.misc.Unsafe.park(Native Method)
java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
java.lang.Thread.run(Thread.java:619)
Thread 23 (main.LruBlockCache.EvictionThread):
State: WAITING
Blocked count: 29
Waited count: 30
Waiting on org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread@6267fe80
Stack:
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:485)
org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:594)
java.lang.Thread.run(Thread.java:619)
Thread 21 (Timer thread for monitoring rpc):
State: TIMED_WAITING
Blocked count: 1
Waited count: 58051
Stack:
java.lang.Object.wait(Native Method)
java.util.TimerThread.mainLoop(Timer.java:509)
java.util.TimerThread.run(Timer.java:462)
Thread 20 (IPC Reader 9 on port 60020):
State: RUNNABLE
Blocked count: 1371
Waited count: 1359
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 19 (IPC Reader 8 on port 60020):
State: RUNNABLE
Blocked count: 1391
Waited count: 1359
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 18 (IPC Reader 7 on port 60020):
State: RUNNABLE
Blocked count: 1371
Waited count: 1359
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 17 (IPC Reader 6 on port 60020):
State: RUNNABLE
Blocked count: 1367
Waited count: 1359
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 16 (IPC Reader 5 on port 60020):
State: RUNNABLE
Blocked count: 1359
Waited count: 1359
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 15 (IPC Reader 4 on port 60020):
State: RUNNABLE
Blocked count: 1361
Waited count: 1359
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 14 (IPC Reader 3 on port 60020):
State: RUNNABLE
Blocked count: 1362
Waited count: 1360
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 13 (IPC Reader 2 on port 60020):
State: RUNNABLE
Blocked count: 1361
Waited count: 1360
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 12 (IPC Reader 1 on port 60020):
State: RUNNABLE
Blocked count: 1362
Waited count: 1360
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 11 (IPC Reader 0 on port 60020):
State: RUNNABLE
Blocked count: 1368
Waited count: 1359
Stack:
sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
java.lang.Thread.run(Thread.java:619)
Thread 5 (Signal Dispatcher):
State: RUNNABLE
Blocked count: 0
Waited count: 0
Stack:
Thread 3 (Finalizer):
State: WAITING
Blocked count: 5664
Waited count: 5626
Waiting on java.lang.ref.ReferenceQueue$Lock@7f81497d
Stack:
java.lang.Object.wait(Native Method)
java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
Thread 2 (Reference Handler):
State: WAITING
Blocked count: 7586
Waited count: 7517
Waiting on java.lang.ref.Reference$Lock@22a010ba
Stack:
java.lang.Object.wait(Native Method)
java.lang.Object.wait(Object.java:485)
java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)