Pastebin launched a little side project called HostCabi.net, check it out ;-)Don't like ads? PRO users don't see any ads ;-)
Guest

busy delete

By: a guest on Jun 20th, 2012  |  syntax: None  |  size: 50.64 KB  |  hits: 36  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. Process Thread Dump:
  2. 73 active threads
  3. Thread 21812 (ResponseProcessor for block blk_-8550997426499413457_56263):
  4.   State: RUNNABLE
  5.   Blocked count: 11
  6.   Waited count: 0
  7.   Stack:
  8.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  9.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  10.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  11.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  12.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  13.     org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
  14.     org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  15.     org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
  16.     org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
  17.     java.io.DataInputStream.readFully(DataInputStream.java:178)
  18.     java.io.DataInputStream.readLong(DataInputStream.java:399)
  19.     org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120)
  20.     org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956)
  21. Thread 21808 (DataStreamer for file /hbase/au3_iu/6f10b2fa1903cf95c066e49556f6192e/.tmp/fc6b741630c3452a84c283da1b354d30 block blk_-8550997426499413457_56263):
  22.   State: TIMED_WAITING
  23.   Blocked count: 7659
  24.   Waited count: 7662
  25.   Stack:
  26.     java.lang.Object.wait(Native Method)
  27.     org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805)
  28. Thread 21805 (IPC Client (47) connection to hmaster101.mentacapital.local/10.10.10.156:8020 from hbase):
  29.   State: TIMED_WAITING
  30.   Blocked count: 25
  31.   Waited count: 26
  32.   Stack:
  33.     java.lang.Object.wait(Native Method)
  34.     org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:680)
  35.     org.apache.hadoop.ipc.Client$Connection.run(Client.java:723)
  36. Thread 21803 (160800141@qtp-941866449-3):
  37.   State: RUNNABLE
  38.   Blocked count: 1
  39.   Waited count: 2
  40.   Stack:
  41.     sun.management.ThreadImpl.getThreadInfo0(Native Method)
  42.     sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
  43.     sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
  44.     org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.java:149)
  45.     org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:697)
  46.     javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
  47.     javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
  48.     org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
  49.     org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
  50.     org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:829)
  51.     org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
  52.     org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
  53.     org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
  54.     org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
  55.     org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
  56.     org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
  57.     org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
  58.     org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
  59.     org.mortbay.jetty.Server.handle(Server.java:326)
  60.     org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
  61. Thread 21786 (ResponseProcessor for block blk_-3607823211736240800_56260):
  62.   State: RUNNABLE
  63.   Blocked count: 2
  64.   Waited count: 0
  65.   Stack:
  66.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  67.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  68.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  69.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  70.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  71.     org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
  72.     org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  73.     org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
  74.     org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
  75.     java.io.DataInputStream.readFully(DataInputStream.java:178)
  76.     java.io.DataInputStream.readLong(DataInputStream.java:399)
  77.     org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120)
  78.     org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956)
  79. Thread 21785 (DataStreamer for file /hbase/.logs/hb105.mentacapital.local.,60020,1339653832110/hb105.mentacapital.local.%2C60020%2C1339653832110.1340233811459 block blk_-3607823211736240800_56260):
  80.   State: TIMED_WAITING
  81.   Blocked count: 7942
  82.   Waited count: 8034
  83.   Stack:
  84.     java.lang.Object.wait(Native Method)
  85.     org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805)
  86. Thread 6885 (regionserver60020-splits-1339824099835):
  87.   State: WAITING
  88.   Blocked count: 127
  89.   Waited count: 209
  90.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@da6426
  91.   Stack:
  92.     sun.misc.Unsafe.park(Native Method)
  93.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  94.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  95.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  96.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  97.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  98.     java.lang.Thread.run(Thread.java:619)
  99. Thread 4956 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-2):
  100.   State: WAITING
  101.   Blocked count: 75
  102.   Waited count: 133
  103.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
  104.   Stack:
  105.     sun.misc.Unsafe.park(Native Method)
  106.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  107.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  108.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  109.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  110.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  111.     java.lang.Thread.run(Thread.java:619)
  112. Thread 4950 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-1):
  113.   State: WAITING
  114.   Blocked count: 100
  115.   Waited count: 180
  116.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
  117.   Stack:
  118.     sun.misc.Unsafe.park(Native Method)
  119.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  120.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  121.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  122.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  123.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  124.     java.lang.Thread.run(Thread.java:619)
  125. Thread 4945 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-0):
  126.   State: WAITING
  127.   Blocked count: 100
  128.   Waited count: 175
  129.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
  130.   Stack:
  131.     sun.misc.Unsafe.park(Native Method)
  132.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  133.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  134.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  135.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  136.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  137.     java.lang.Thread.run(Thread.java:619)
  138. Thread 2743 (regionserver60020-smallCompactions-1339717999356):
  139.   State: WAITING
  140.   Blocked count: 137870
  141.   Waited count: 5766
  142.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3e6ee7e4
  143.   Stack:
  144.     sun.misc.Unsafe.park(Native Method)
  145.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  146.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  147.     java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:220)
  148.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  149.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  150.     java.lang.Thread.run(Thread.java:619)
  151. Thread 831 (regionserver60020-largeCompactions-1339663832452):
  152.   State: RUNNABLE
  153.   Blocked count: 496027
  154.   Waited count: 3044
  155.   Stack:
  156.     org.apache.hadoop.hbase.regionserver.StoreFileScanner.peek(StoreFileScanner.java:121)
  157.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:96)
  158.     org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:368)
  159.     org.apache.hadoop.hbase.regionserver.Store.compactStore(Store.java:1528)
  160.     org.apache.hadoop.hbase.regionserver.Store.compact(Store.java:988)
  161.     org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1188)
  162.     org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.run(CompactionRequest.java:187)
  163.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  164.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  165.     java.lang.Thread.run(Thread.java:619)
  166. Thread 202 (sendParams-1):
  167.   State: TIMED_WAITING
  168.   Blocked count: 7
  169.   Waited count: 31437
  170.   Stack:
  171.     sun.misc.Unsafe.park(Native Method)
  172.     java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  173.     java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
  174.     java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
  175.     java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
  176.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
  177.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  178.     java.lang.Thread.run(Thread.java:619)
  179. Thread 94 (PostOpenDeployTasks:69eaa8adf8dab4c1e28905ce6eb726f7-EventThread):
  180.   State: WAITING
  181.   Blocked count: 0
  182.   Waited count: 4
  183.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59c87031
  184.   Stack:
  185.     sun.misc.Unsafe.park(Native Method)
  186.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  187.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  188.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  189.     org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493)
  190. Thread 93 (PostOpenDeployTasks:69eaa8adf8dab4c1e28905ce6eb726f7-SendThread(hmaster101.mentacapital.local.:2181)):
  191.   State: RUNNABLE
  192.   Blocked count: 2
  193.   Waited count: 0
  194.   Stack:
  195.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  196.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  197.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  198.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  199.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  200.     org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274)
  201.     org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035)
  202. Thread 83 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-2):
  203.   State: WAITING
  204.   Blocked count: 612
  205.   Waited count: 860
  206.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
  207.   Stack:
  208.     sun.misc.Unsafe.park(Native Method)
  209.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  210.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  211.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  212.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  213.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  214.     java.lang.Thread.run(Thread.java:619)
  215. Thread 82 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-1):
  216.   State: WAITING
  217.   Blocked count: 605
  218.   Waited count: 859
  219.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
  220.   Stack:
  221.     sun.misc.Unsafe.park(Native Method)
  222.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  223.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  224.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  225.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  226.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  227.     java.lang.Thread.run(Thread.java:619)
  228. Thread 81 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-0):
  229.   State: WAITING
  230.   Blocked count: 581
  231.   Waited count: 825
  232.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
  233.   Stack:
  234.     sun.misc.Unsafe.park(Native Method)
  235.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  236.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  237.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  238.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  239.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  240.     java.lang.Thread.run(Thread.java:619)
  241. Thread 69 (SplitLogWorker-hb105.mentacapital.local.,60020,1339653832110):
  242.   State: WAITING
  243.   Blocked count: 4
  244.   Waited count: 3
  245.   Waiting on java.lang.Object@2e39060b
  246.   Stack:
  247.     java.lang.Object.wait(Native Method)
  248.     java.lang.Object.wait(Object.java:485)
  249.     org.apache.hadoop.hbase.regionserver.SplitLogWorker.taskLoop(SplitLogWorker.java:205)
  250.     org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:165)
  251.     java.lang.Thread.run(Thread.java:619)
  252. Thread 68 (PRI IPC Server handler 9 on 60020):
  253.   State: WAITING
  254.   Blocked count: 29
  255.   Waited count: 91
  256.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  257.   Stack:
  258.     sun.misc.Unsafe.park(Native Method)
  259.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  260.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  261.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  262.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  263. Thread 67 (PRI IPC Server handler 8 on 60020):
  264.   State: WAITING
  265.   Blocked count: 32
  266.   Waited count: 94
  267.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  268.   Stack:
  269.     sun.misc.Unsafe.park(Native Method)
  270.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  271.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  272.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  273.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  274. Thread 66 (PRI IPC Server handler 7 on 60020):
  275.   State: WAITING
  276.   Blocked count: 30
  277.   Waited count: 88
  278.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  279.   Stack:
  280.     sun.misc.Unsafe.park(Native Method)
  281.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  282.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  283.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  284.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  285. Thread 65 (PRI IPC Server handler 6 on 60020):
  286.   State: WAITING
  287.   Blocked count: 28
  288.   Waited count: 90
  289.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  290.   Stack:
  291.     sun.misc.Unsafe.park(Native Method)
  292.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  293.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  294.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  295.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  296. Thread 64 (PRI IPC Server handler 5 on 60020):
  297.   State: WAITING
  298.   Blocked count: 28
  299.   Waited count: 87
  300.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  301.   Stack:
  302.     sun.misc.Unsafe.park(Native Method)
  303.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  304.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  305.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  306.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  307. Thread 63 (PRI IPC Server handler 4 on 60020):
  308.   State: WAITING
  309.   Blocked count: 36
  310.   Waited count: 89
  311.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  312.   Stack:
  313.     sun.misc.Unsafe.park(Native Method)
  314.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  315.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  316.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  317.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  318. Thread 62 (PRI IPC Server handler 3 on 60020):
  319.   State: WAITING
  320.   Blocked count: 34
  321.   Waited count: 88
  322.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  323.   Stack:
  324.     sun.misc.Unsafe.park(Native Method)
  325.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  326.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  327.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  328.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  329. Thread 61 (PRI IPC Server handler 2 on 60020):
  330.   State: WAITING
  331.   Blocked count: 48
  332.   Waited count: 100
  333.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  334.   Stack:
  335.     sun.misc.Unsafe.park(Native Method)
  336.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  337.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  338.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  339.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  340. Thread 60 (PRI IPC Server handler 1 on 60020):
  341.   State: WAITING
  342.   Blocked count: 34
  343.   Waited count: 91
  344.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  345.   Stack:
  346.     sun.misc.Unsafe.park(Native Method)
  347.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  348.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  349.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  350.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  351. Thread 59 (PRI IPC Server handler 0 on 60020):
  352.   State: WAITING
  353.   Blocked count: 49
  354.   Waited count: 106
  355.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  356.   Stack:
  357.     sun.misc.Unsafe.park(Native Method)
  358.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  359.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  360.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  361.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  362. Thread 58 (IPC Server handler 9 on 60020):
  363.   State: RUNNABLE
  364.   Blocked count: 130980
  365.   Waited count: 947471
  366.   Stack:
  367.     org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
  368.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
  369.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
  370.     org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
  371.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
  372.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
  373.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
  374.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
  375.     org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
  376.     org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
  377.     org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
  378.     org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
  379.     org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
  380.     org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
  381.     sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  382.     sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  383.     java.lang.reflect.Method.invoke(Method.java:597)
  384.     org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  385.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
  386. Thread 57 (IPC Server handler 8 on 60020):
  387.   State: WAITING
  388.   Blocked count: 117086
  389.   Waited count: 934181
  390.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  391.   Stack:
  392.     sun.misc.Unsafe.park(Native Method)
  393.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  394.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  395.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  396.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  397. Thread 56 (IPC Server handler 7 on 60020):
  398.   State: WAITING
  399.   Blocked count: 105514
  400.   Waited count: 922107
  401.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  402.   Stack:
  403.     sun.misc.Unsafe.park(Native Method)
  404.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  405.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  406.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  407.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  408. Thread 55 (IPC Server handler 6 on 60020):
  409.   State: WAITING
  410.   Blocked count: 102307
  411.   Waited count: 918723
  412.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  413.   Stack:
  414.     sun.misc.Unsafe.park(Native Method)
  415.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  416.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  417.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  418.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  419. Thread 54 (IPC Server handler 5 on 60020):
  420.   State: WAITING
  421.   Blocked count: 121432
  422.   Waited count: 938055
  423.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  424.   Stack:
  425.     sun.misc.Unsafe.park(Native Method)
  426.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  427.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  428.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  429.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  430. Thread 53 (IPC Server handler 4 on 60020):
  431.   State: RUNNABLE
  432.   Blocked count: 130846
  433.   Waited count: 946687
  434.   Stack:
  435.     org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
  436.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
  437.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
  438.     org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
  439.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
  440.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
  441.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
  442.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
  443.     org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
  444.     org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
  445.     org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
  446.     org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
  447.     org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
  448.     org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
  449.     sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  450.     sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  451.     java.lang.reflect.Method.invoke(Method.java:597)
  452.     org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  453.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
  454. Thread 52 (IPC Server handler 3 on 60020):
  455.   State: RUNNABLE
  456.   Blocked count: 113158
  457.   Waited count: 930380
  458.   Stack:
  459.     org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.blockSeek(HFileReaderV2.java:802)
  460.     org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.loadBlockAndSeekToKey(HFileReaderV2.java:710)
  461.     org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:461)
  462.     org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:475)
  463.     org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:226)
  464.     org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:145)
  465.     org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:130)
  466.     org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2001)
  467.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3274)
  468.     org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1604)
  469.     org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1596)
  470.     org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1572)
  471.     org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4065)
  472.     org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4039)
  473.     org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1944)
  474.     org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3346)
  475.     sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  476.     sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  477.     java.lang.reflect.Method.invoke(Method.java:597)
  478.     org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  479. Thread 51 (IPC Server handler 2 on 60020):
  480.   State: WAITING
  481.   Blocked count: 113506
  482.   Waited count: 930486
  483.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  484.   Stack:
  485.     sun.misc.Unsafe.park(Native Method)
  486.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  487.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  488.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  489.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  490. Thread 50 (IPC Server handler 1 on 60020):
  491.   State: RUNNABLE
  492.   Blocked count: 113476
  493.   Waited count: 928781
  494.   Stack:
  495.     org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
  496.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
  497.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
  498.     org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
  499.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
  500.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
  501.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
  502.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
  503.     org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
  504.     org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
  505.     org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
  506.     org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
  507.     org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
  508.     org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
  509.     sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  510.     sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  511.     java.lang.reflect.Method.invoke(Method.java:597)
  512.     org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  513.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
  514. Thread 49 (IPC Server handler 0 on 60020):
  515.   State: RUNNABLE
  516.   Blocked count: 121487
  517.   Waited count: 936162
  518.   Stack:
  519.     org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
  520.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
  521.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
  522.     org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
  523.     org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
  524.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
  525.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
  526.     org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
  527.     org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
  528.     org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
  529.     org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
  530.     org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
  531.     org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
  532.     org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
  533.     sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  534.     sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  535.     java.lang.reflect.Method.invoke(Method.java:597)
  536.     org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  537.     org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
  538. Thread 10 (IPC Server listener on 60020):
  539.   State: RUNNABLE
  540.   Blocked count: 13590
  541.   Waited count: 0
  542.   Stack:
  543.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  544.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  545.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  546.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  547.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  548.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  549.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:610)
  550. Thread 22 (IPC Server Responder):
  551.   State: RUNNABLE
  552.   Blocked count: 6533
  553.   Waited count: 6452
  554.   Stack:
  555.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  556.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  557.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  558.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  559.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  560.     org.apache.hadoop.hbase.ipc.HBaseServer$Responder.doRunLoop(HBaseServer.java:799)
  561.     org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:782)
  562. Thread 48 (Timer-0):
  563.   State: TIMED_WAITING
  564.   Blocked count: 1
  565.   Waited count: 19351
  566.   Stack:
  567.     java.lang.Object.wait(Native Method)
  568.     java.util.TimerThread.mainLoop(Timer.java:509)
  569.     java.util.TimerThread.run(Timer.java:462)
  570. Thread 47 (1136341612@qtp-941866449-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030):
  571.   State: RUNNABLE
  572.   Blocked count: 1
  573.   Waited count: 1
  574.   Stack:
  575.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  576.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  577.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  578.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  579.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  580.     org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
  581.     org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
  582.     org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
  583.     org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
  584.     org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
  585. Thread 32 (regionserver60020.leaseChecker):
  586.   State: TIMED_WAITING
  587.   Blocked count: 0
  588.   Waited count: 4559366
  589.   Stack:
  590.     sun.misc.Unsafe.park(Native Method)
  591.     java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  592.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
  593.     java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
  594.     org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:83)
  595.     java.lang.Thread.run(Thread.java:619)
  596. Thread 31 (regionserver60020.compactionChecker):
  597.   State: TIMED_WAITING
  598.   Blocked count: 75
  599.   Waited count: 60
  600.   Stack:
  601.     java.lang.Object.wait(Native Method)
  602.     org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
  603.     org.apache.hadoop.hbase.Chore.run(Chore.java:75)
  604.     java.lang.Thread.run(Thread.java:619)
  605. Thread 30 (regionserver60020.cacheFlusher):
  606.   State: TIMED_WAITING
  607.   Blocked count: 40438
  608.   Waited count: 70695
  609.   Stack:
  610.     sun.misc.Unsafe.park(Native Method)
  611.     java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  612.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
  613.     java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
  614.     java.util.concurrent.DelayQueue.poll(DelayQueue.java:39)
  615.     org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:218)
  616.     java.lang.Thread.run(Thread.java:619)
  617. Thread 38 (regionserver60020.logRoller):
  618.   State: TIMED_WAITING
  619.   Blocked count: 1226
  620.   Waited count: 59826
  621.   Stack:
  622.     java.lang.Object.wait(Native Method)
  623.     org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:76)
  624.     java.lang.Thread.run(Thread.java:619)
  625. Thread 44 (Timer thread for monitoring jvm):
  626.   State: TIMED_WAITING
  627.   Blocked count: 4
  628.   Waited count: 58050
  629.   Stack:
  630.     java.lang.Object.wait(Native Method)
  631.     java.util.TimerThread.mainLoop(Timer.java:509)
  632.     java.util.TimerThread.run(Timer.java:462)
  633. Thread 43 (Timer thread for monitoring hbase):
  634.   State: TIMED_WAITING
  635.   Blocked count: 3
  636.   Waited count: 58050
  637.   Stack:
  638.     java.lang.Object.wait(Native Method)
  639.     java.util.TimerThread.mainLoop(Timer.java:509)
  640.     java.util.TimerThread.run(Timer.java:462)
  641. Thread 42 (regionserver60020.decayingSampleTick.1):
  642.   State: TIMED_WAITING
  643.   Blocked count: 0
  644.   Waited count: 1160870
  645.   Stack:
  646.     sun.misc.Unsafe.park(Native Method)
  647.     java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  648.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
  649.     java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
  650.     java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
  651.     java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
  652.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  653.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  654.     java.lang.Thread.run(Thread.java:619)
  655. Thread 41 (regionserver60020.logSyncer):
  656.   State: TIMED_WAITING
  657.   Blocked count: 2698
  658.   Waited count: 582724
  659.   Stack:
  660.     java.lang.Thread.sleep(Native Method)
  661.     org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1193)
  662.     java.lang.Thread.run(Thread.java:619)
  663. Thread 40 (LeaseChecker):
  664.   State: TIMED_WAITING
  665.   Blocked count: 19346
  666.   Waited count: 618946
  667.   Stack:
  668.     java.lang.Thread.sleep(Native Method)
  669.     org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1302)
  670.     java.lang.Thread.run(Thread.java:619)
  671. Thread 37 (DestroyJavaVM):
  672.   State: RUNNABLE
  673.   Blocked count: 0
  674.   Waited count: 0
  675.   Stack:
  676. Thread 33 (IPC Client (47) connection to hmaster101.mentacapital.local./10.10.10.156:60000 from hbase):
  677.   State: TIMED_WAITING
  678.   Blocked count: 193043
  679.   Waited count: 193043
  680.   Stack:
  681.     java.lang.Object.wait(Native Method)
  682.     org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:459)
  683.     org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:504)
  684. Thread 29 (regionserver60020-EventThread):
  685.   State: WAITING
  686.   Blocked count: 4
  687.   Waited count: 3
  688.   Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3003e926
  689.   Stack:
  690.     sun.misc.Unsafe.park(Native Method)
  691.     java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  692.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  693.     java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  694.     org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493)
  695. Thread 28 (regionserver60020-SendThread(hmaster101.mentacapital.local.:2181)):
  696.   State: RUNNABLE
  697.   Blocked count: 240
  698.   Waited count: 0
  699.   Stack:
  700.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  701.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  702.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  703.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  704.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  705.     org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274)
  706.     org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035)
  707. Thread 26 (regionserver60020):
  708.   State: TIMED_WAITING
  709.   Blocked count: 193160
  710.   Waited count: 386109
  711.   Stack:
  712.     java.lang.Object.wait(Native Method)
  713.     org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
  714.     org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:55)
  715.     org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:714)
  716.     java.lang.Thread.run(Thread.java:619)
  717. Thread 25 (LRU Statistics #0):
  718.   State: TIMED_WAITING
  719.   Blocked count: 0
  720.   Waited count: 1942
  721.   Stack:
  722.     sun.misc.Unsafe.park(Native Method)
  723.     java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  724.     java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
  725.     java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
  726.     java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
  727.     java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
  728.     java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  729.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  730.     java.lang.Thread.run(Thread.java:619)
  731. Thread 23 (main.LruBlockCache.EvictionThread):
  732.   State: WAITING
  733.   Blocked count: 29
  734.   Waited count: 30
  735.   Waiting on org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread@6267fe80
  736.   Stack:
  737.     java.lang.Object.wait(Native Method)
  738.     java.lang.Object.wait(Object.java:485)
  739.     org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:594)
  740.     java.lang.Thread.run(Thread.java:619)
  741. Thread 21 (Timer thread for monitoring rpc):
  742.   State: TIMED_WAITING
  743.   Blocked count: 1
  744.   Waited count: 58051
  745.   Stack:
  746.     java.lang.Object.wait(Native Method)
  747.     java.util.TimerThread.mainLoop(Timer.java:509)
  748.     java.util.TimerThread.run(Timer.java:462)
  749. Thread 20 (IPC Reader 9 on port 60020):
  750.   State: RUNNABLE
  751.   Blocked count: 1371
  752.   Waited count: 1359
  753.   Stack:
  754.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  755.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  756.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  757.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  758.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  759.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  760.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  761.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  762.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  763.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  764.     java.lang.Thread.run(Thread.java:619)
  765. Thread 19 (IPC Reader 8 on port 60020):
  766.   State: RUNNABLE
  767.   Blocked count: 1391
  768.   Waited count: 1359
  769.   Stack:
  770.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  771.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  772.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  773.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  774.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  775.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  776.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  777.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  778.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  779.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  780.     java.lang.Thread.run(Thread.java:619)
  781. Thread 18 (IPC Reader 7 on port 60020):
  782.   State: RUNNABLE
  783.   Blocked count: 1371
  784.   Waited count: 1359
  785.   Stack:
  786.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  787.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  788.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  789.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  790.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  791.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  792.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  793.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  794.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  795.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  796.     java.lang.Thread.run(Thread.java:619)
  797. Thread 17 (IPC Reader 6 on port 60020):
  798.   State: RUNNABLE
  799.   Blocked count: 1367
  800.   Waited count: 1359
  801.   Stack:
  802.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  803.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  804.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  805.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  806.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  807.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  808.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  809.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  810.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  811.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  812.     java.lang.Thread.run(Thread.java:619)
  813. Thread 16 (IPC Reader 5 on port 60020):
  814.   State: RUNNABLE
  815.   Blocked count: 1359
  816.   Waited count: 1359
  817.   Stack:
  818.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  819.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  820.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  821.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  822.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  823.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  824.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  825.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  826.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  827.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  828.     java.lang.Thread.run(Thread.java:619)
  829. Thread 15 (IPC Reader 4 on port 60020):
  830.   State: RUNNABLE
  831.   Blocked count: 1361
  832.   Waited count: 1359
  833.   Stack:
  834.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  835.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  836.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  837.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  838.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  839.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  840.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  841.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  842.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  843.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  844.     java.lang.Thread.run(Thread.java:619)
  845. Thread 14 (IPC Reader 3 on port 60020):
  846.   State: RUNNABLE
  847.   Blocked count: 1362
  848.   Waited count: 1360
  849.   Stack:
  850.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  851.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  852.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  853.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  854.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  855.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  856.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  857.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  858.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  859.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  860.     java.lang.Thread.run(Thread.java:619)
  861. Thread 13 (IPC Reader 2 on port 60020):
  862.   State: RUNNABLE
  863.   Blocked count: 1361
  864.   Waited count: 1360
  865.   Stack:
  866.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  867.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  868.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  869.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  870.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  871.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  872.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  873.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  874.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  875.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  876.     java.lang.Thread.run(Thread.java:619)
  877. Thread 12 (IPC Reader 1 on port 60020):
  878.   State: RUNNABLE
  879.   Blocked count: 1362
  880.   Waited count: 1360
  881.   Stack:
  882.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  883.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  884.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  885.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  886.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  887.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  888.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  889.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  890.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  891.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  892.     java.lang.Thread.run(Thread.java:619)
  893. Thread 11 (IPC Reader 0 on port 60020):
  894.   State: RUNNABLE
  895.   Blocked count: 1368
  896.   Waited count: 1359
  897.   Stack:
  898.     sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  899.     sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  900.     sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  901.     sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  902.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  903.     sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  904.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  905.     org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  906.     java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  907.     java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  908.     java.lang.Thread.run(Thread.java:619)
  909. Thread 5 (Signal Dispatcher):
  910.   State: RUNNABLE
  911.   Blocked count: 0
  912.   Waited count: 0
  913.   Stack:
  914. Thread 3 (Finalizer):
  915.   State: WAITING
  916.   Blocked count: 5664
  917.   Waited count: 5626
  918.   Waiting on java.lang.ref.ReferenceQueue$Lock@7f81497d
  919.   Stack:
  920.     java.lang.Object.wait(Native Method)
  921.     java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
  922.     java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
  923.     java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
  924. Thread 2 (Reference Handler):
  925.   State: WAITING
  926.   Blocked count: 7586
  927.   Waited count: 7517
  928.   Waiting on java.lang.ref.Reference$Lock@22a010ba
  929.   Stack:
  930.     java.lang.Object.wait(Native Method)
  931.     java.lang.Object.wait(Object.java:485)
  932.     java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)