API tools faq
paste
Advertisement
Guest User

busy delete

a guest
Jun 20th, 2012
160
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 50.64 KB | None | 0 0
raw download clone embed print report
  1. Process Thread Dump:
  2. 73 active threads
  3. Thread 21812 (ResponseProcessor for block blk_-8550997426499413457_56263):
  4. State: RUNNABLE
  5. Blocked count: 11
  6. Waited count: 0
  7. Stack:
  8. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  9. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  10. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  11. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  12. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  13. org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
  14. org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  15. org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
  16. org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
  17. java.io.DataInputStream.readFully(DataInputStream.java:178)
  18. java.io.DataInputStream.readLong(DataInputStream.java:399)
  19. org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120)
  20. org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956)
  21. Thread 21808 (DataStreamer for file /hbase/au3_iu/6f10b2fa1903cf95c066e49556f6192e/.tmp/fc6b741630c3452a84c283da1b354d30 block blk_-8550997426499413457_56263):
  22. State: TIMED_WAITING
  23. Blocked count: 7659
  24. Waited count: 7662
  25. Stack:
  26. java.lang.Object.wait(Native Method)
  27. org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805)
  28. Thread 21805 (IPC Client (47) connection to hmaster101.mentacapital.local/10.10.10.156:8020 from hbase):
  29. State: TIMED_WAITING
  30. Blocked count: 25
  31. Waited count: 26
  32. Stack:
  33. java.lang.Object.wait(Native Method)
  34. org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:680)
  35. org.apache.hadoop.ipc.Client$Connection.run(Client.java:723)
  36. Thread 21803 (160800141@qtp-941866449-3):
  37. State: RUNNABLE
  38. Blocked count: 1
  39. Waited count: 2
  40. Stack:
  41. sun.management.ThreadImpl.getThreadInfo0(Native Method)
  42. sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:147)
  43. sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:123)
  44. org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.java:149)
  45. org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:697)
  46. javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
  47. javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
  48. org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
  49. org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
  50. org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:829)
  51. org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
  52. org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
  53. org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
  54. org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
  55. org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
  56. org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
  57. org.mortbay.jetty.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:230)
  58. org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
  59. org.mortbay.jetty.Server.handle(Server.java:326)
  60. org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
  61. Thread 21786 (ResponseProcessor for block blk_-3607823211736240800_56260):
  62. State: RUNNABLE
  63. Blocked count: 2
  64. Waited count: 0
  65. Stack:
  66. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  67. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  68. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  69. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  70. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  71. org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
  72. org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  73. org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
  74. org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
  75. java.io.DataInputStream.readFully(DataInputStream.java:178)
  76. java.io.DataInputStream.readLong(DataInputStream.java:399)
  77. org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:120)
  78. org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2956)
  79. Thread 21785 (DataStreamer for file /hbase/.logs/hb105.mentacapital.local.,60020,1339653832110/hb105.mentacapital.local.%2C60020%2C1339653832110.1340233811459 block blk_-3607823211736240800_56260):
  80. State: TIMED_WAITING
  81. Blocked count: 7942
  82. Waited count: 8034
  83. Stack:
  84. java.lang.Object.wait(Native Method)
  85. org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2805)
  86. Thread 6885 (regionserver60020-splits-1339824099835):
  87. State: WAITING
  88. Blocked count: 127
  89. Waited count: 209
  90. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@da6426
  91. Stack:
  92. sun.misc.Unsafe.park(Native Method)
  93. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  94. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  95. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  96. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  97. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  98. java.lang.Thread.run(Thread.java:619)
  99. Thread 4956 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-2):
  100. State: WAITING
  101. Blocked count: 75
  102. Waited count: 133
  103. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
  104. Stack:
  105. sun.misc.Unsafe.park(Native Method)
  106. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  107. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  108. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  109. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  110. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  111. java.lang.Thread.run(Thread.java:619)
  112. Thread 4950 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-1):
  113. State: WAITING
  114. Blocked count: 100
  115. Waited count: 180
  116. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
  117. Stack:
  118. sun.misc.Unsafe.park(Native Method)
  119. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  120. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  121. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  122. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  123. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  124. java.lang.Thread.run(Thread.java:619)
  125. Thread 4945 (RS_CLOSE_REGION-hb105.mentacapital.local.,60020,1339653832110-0):
  126. State: WAITING
  127. Blocked count: 100
  128. Waited count: 175
  129. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@e27298
  130. Stack:
  131. sun.misc.Unsafe.park(Native Method)
  132. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  133. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  134. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  135. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  136. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  137. java.lang.Thread.run(Thread.java:619)
  138. Thread 2743 (regionserver60020-smallCompactions-1339717999356):
  139. State: WAITING
  140. Blocked count: 137870
  141. Waited count: 5766
  142. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3e6ee7e4
  143. Stack:
  144. sun.misc.Unsafe.park(Native Method)
  145. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  146. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  147. java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:220)
  148. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  149. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  150. java.lang.Thread.run(Thread.java:619)
  151. Thread 831 (regionserver60020-largeCompactions-1339663832452):
  152. State: RUNNABLE
  153. Blocked count: 496027
  154. Waited count: 3044
  155. Stack:
  156. org.apache.hadoop.hbase.regionserver.StoreFileScanner.peek(StoreFileScanner.java:121)
  157. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:96)
  158. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:368)
  159. org.apache.hadoop.hbase.regionserver.Store.compactStore(Store.java:1528)
  160. org.apache.hadoop.hbase.regionserver.Store.compact(Store.java:988)
  161. org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1188)
  162. org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest.run(CompactionRequest.java:187)
  163. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  164. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  165. java.lang.Thread.run(Thread.java:619)
  166. Thread 202 (sendParams-1):
  167. State: TIMED_WAITING
  168. Blocked count: 7
  169. Waited count: 31437
  170. Stack:
  171. sun.misc.Unsafe.park(Native Method)
  172. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  173. java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
  174. java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
  175. java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
  176. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
  177. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  178. java.lang.Thread.run(Thread.java:619)
  179. Thread 94 (PostOpenDeployTasks:69eaa8adf8dab4c1e28905ce6eb726f7-EventThread):
  180. State: WAITING
  181. Blocked count: 0
  182. Waited count: 4
  183. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@59c87031
  184. Stack:
  185. sun.misc.Unsafe.park(Native Method)
  186. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  187. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  188. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  189. org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493)
  190. Thread 93 (PostOpenDeployTasks:69eaa8adf8dab4c1e28905ce6eb726f7-SendThread(hmaster101.mentacapital.local.:2181)):
  191. State: RUNNABLE
  192. Blocked count: 2
  193. Waited count: 0
  194. Stack:
  195. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  196. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  197. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  198. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  199. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  200. org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274)
  201. org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035)
  202. Thread 83 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-2):
  203. State: WAITING
  204. Blocked count: 612
  205. Waited count: 860
  206. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
  207. Stack:
  208. sun.misc.Unsafe.park(Native Method)
  209. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  210. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  211. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  212. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  213. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  214. java.lang.Thread.run(Thread.java:619)
  215. Thread 82 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-1):
  216. State: WAITING
  217. Blocked count: 605
  218. Waited count: 859
  219. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
  220. Stack:
  221. sun.misc.Unsafe.park(Native Method)
  222. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  223. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  224. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  225. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  226. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  227. java.lang.Thread.run(Thread.java:619)
  228. Thread 81 (RS_OPEN_REGION-hb105.mentacapital.local.,60020,1339653832110-0):
  229. State: WAITING
  230. Blocked count: 581
  231. Waited count: 825
  232. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@47516490
  233. Stack:
  234. sun.misc.Unsafe.park(Native Method)
  235. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  236. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  237. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  238. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  239. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  240. java.lang.Thread.run(Thread.java:619)
  241. Thread 69 (SplitLogWorker-hb105.mentacapital.local.,60020,1339653832110):
  242. State: WAITING
  243. Blocked count: 4
  244. Waited count: 3
  245. Waiting on java.lang.Object@2e39060b
  246. Stack:
  247. java.lang.Object.wait(Native Method)
  248. java.lang.Object.wait(Object.java:485)
  249. org.apache.hadoop.hbase.regionserver.SplitLogWorker.taskLoop(SplitLogWorker.java:205)
  250. org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:165)
  251. java.lang.Thread.run(Thread.java:619)
  252. Thread 68 (PRI IPC Server handler 9 on 60020):
  253. State: WAITING
  254. Blocked count: 29
  255. Waited count: 91
  256. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  257. Stack:
  258. sun.misc.Unsafe.park(Native Method)
  259. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  260. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  261. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  262. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  263. Thread 67 (PRI IPC Server handler 8 on 60020):
  264. State: WAITING
  265. Blocked count: 32
  266. Waited count: 94
  267. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  268. Stack:
  269. sun.misc.Unsafe.park(Native Method)
  270. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  271. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  272. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  273. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  274. Thread 66 (PRI IPC Server handler 7 on 60020):
  275. State: WAITING
  276. Blocked count: 30
  277. Waited count: 88
  278. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  279. Stack:
  280. sun.misc.Unsafe.park(Native Method)
  281. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  282. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  283. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  284. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  285. Thread 65 (PRI IPC Server handler 6 on 60020):
  286. State: WAITING
  287. Blocked count: 28
  288. Waited count: 90
  289. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  290. Stack:
  291. sun.misc.Unsafe.park(Native Method)
  292. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  293. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  294. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  295. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  296. Thread 64 (PRI IPC Server handler 5 on 60020):
  297. State: WAITING
  298. Blocked count: 28
  299. Waited count: 87
  300. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  301. Stack:
  302. sun.misc.Unsafe.park(Native Method)
  303. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  304. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  305. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  306. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  307. Thread 63 (PRI IPC Server handler 4 on 60020):
  308. State: WAITING
  309. Blocked count: 36
  310. Waited count: 89
  311. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  312. Stack:
  313. sun.misc.Unsafe.park(Native Method)
  314. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  315. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  316. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  317. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  318. Thread 62 (PRI IPC Server handler 3 on 60020):
  319. State: WAITING
  320. Blocked count: 34
  321. Waited count: 88
  322. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  323. Stack:
  324. sun.misc.Unsafe.park(Native Method)
  325. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  326. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  327. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  328. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  329. Thread 61 (PRI IPC Server handler 2 on 60020):
  330. State: WAITING
  331. Blocked count: 48
  332. Waited count: 100
  333. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  334. Stack:
  335. sun.misc.Unsafe.park(Native Method)
  336. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  337. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  338. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  339. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  340. Thread 60 (PRI IPC Server handler 1 on 60020):
  341. State: WAITING
  342. Blocked count: 34
  343. Waited count: 91
  344. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  345. Stack:
  346. sun.misc.Unsafe.park(Native Method)
  347. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  348. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  349. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  350. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  351. Thread 59 (PRI IPC Server handler 0 on 60020):
  352. State: WAITING
  353. Blocked count: 49
  354. Waited count: 106
  355. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@21bbd3e2
  356. Stack:
  357. sun.misc.Unsafe.park(Native Method)
  358. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  359. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  360. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  361. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  362. Thread 58 (IPC Server handler 9 on 60020):
  363. State: RUNNABLE
  364. Blocked count: 130980
  365. Waited count: 947471
  366. Stack:
  367. org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
  368. org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
  369. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
  370. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
  371. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
  372. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
  373. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
  374. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
  375. org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
  376. org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
  377. org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
  378. org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
  379. org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
  380. org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
  381. sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  382. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  383. java.lang.reflect.Method.invoke(Method.java:597)
  384. org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  385. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
  386. Thread 57 (IPC Server handler 8 on 60020):
  387. State: WAITING
  388. Blocked count: 117086
  389. Waited count: 934181
  390. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  391. Stack:
  392. sun.misc.Unsafe.park(Native Method)
  393. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  394. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  395. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  396. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  397. Thread 56 (IPC Server handler 7 on 60020):
  398. State: WAITING
  399. Blocked count: 105514
  400. Waited count: 922107
  401. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  402. Stack:
  403. sun.misc.Unsafe.park(Native Method)
  404. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  405. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  406. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  407. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  408. Thread 55 (IPC Server handler 6 on 60020):
  409. State: WAITING
  410. Blocked count: 102307
  411. Waited count: 918723
  412. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  413. Stack:
  414. sun.misc.Unsafe.park(Native Method)
  415. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  416. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  417. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  418. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  419. Thread 54 (IPC Server handler 5 on 60020):
  420. State: WAITING
  421. Blocked count: 121432
  422. Waited count: 938055
  423. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  424. Stack:
  425. sun.misc.Unsafe.park(Native Method)
  426. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  427. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  428. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  429. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  430. Thread 53 (IPC Server handler 4 on 60020):
  431. State: RUNNABLE
  432. Blocked count: 130846
  433. Waited count: 946687
  434. Stack:
  435. org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
  436. org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
  437. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
  438. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
  439. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
  440. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
  441. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
  442. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
  443. org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
  444. org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
  445. org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
  446. org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
  447. org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
  448. org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
  449. sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  450. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  451. java.lang.reflect.Method.invoke(Method.java:597)
  452. org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  453. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
  454. Thread 52 (IPC Server handler 3 on 60020):
  455. State: RUNNABLE
  456. Blocked count: 113158
  457. Waited count: 930380
  458. Stack:
  459. org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.blockSeek(HFileReaderV2.java:802)
  460. org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.loadBlockAndSeekToKey(HFileReaderV2.java:710)
  461. org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:461)
  462. org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:475)
  463. org.apache.hadoop.hbase.regionserver.StoreFileScanner.seekAtOrAfter(StoreFileScanner.java:226)
  464. org.apache.hadoop.hbase.regionserver.StoreFileScanner.seek(StoreFileScanner.java:145)
  465. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:130)
  466. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2001)
  467. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3274)
  468. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1604)
  469. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1596)
  470. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1572)
  471. org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4065)
  472. org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4039)
  473. org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1944)
  474. org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3346)
  475. sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  476. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  477. java.lang.reflect.Method.invoke(Method.java:597)
  478. org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  479. Thread 51 (IPC Server handler 2 on 60020):
  480. State: WAITING
  481. Blocked count: 113506
  482. Waited count: 930486
  483. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@132f9bcd
  484. Stack:
  485. sun.misc.Unsafe.park(Native Method)
  486. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  487. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  488. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  489. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1348)
  490. Thread 50 (IPC Server handler 1 on 60020):
  491. State: RUNNABLE
  492. Blocked count: 113476
  493. Waited count: 928781
  494. Stack:
  495. org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
  496. org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
  497. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
  498. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
  499. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
  500. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
  501. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
  502. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
  503. org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
  504. org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
  505. org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
  506. org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
  507. org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
  508. org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
  509. sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  510. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  511. java.lang.reflect.Method.invoke(Method.java:597)
  512. org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  513. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
  514. Thread 49 (IPC Server handler 0 on 60020):
  515. State: RUNNABLE
  516. Blocked count: 121487
  517. Waited count: 936162
  518. Stack:
  519. org.apache.hadoop.hbase.regionserver.StoreFileScanner.realSeekDone(StoreFileScanner.java:340)
  520. org.apache.hadoop.hbase.regionserver.KeyValueHeap.pollRealKV(KeyValueHeap.java:331)
  521. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:105)
  522. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:406)
  523. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:127)
  524. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3354)
  525. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3310)
  526. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.next(HRegion.java:3327)
  527. org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4066)
  528. org.apache.hadoop.hbase.regionserver.HRegion.prepareDeleteTimestamps(HRegion.java:1710)
  529. org.apache.hadoop.hbase.regionserver.HRegion.internalDelete(HRegion.java:1753)
  530. org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1651)
  531. org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:2513)
  532. org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:3343)
  533. sun.reflect.GeneratedMethodAccessor26.invoke(Unknown Source)
  534. sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  535. java.lang.reflect.Method.invoke(Method.java:597)
  536. org.apache.hadoop.hbase.ipc.WritableRpcEngine$Server.call(WritableRpcEngine.java:364)
  537. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1376)
  538. Thread 10 (IPC Server listener on 60020):
  539. State: RUNNABLE
  540. Blocked count: 13590
  541. Waited count: 0
  542. Stack:
  543. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  544. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  545. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  546. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  547. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  548. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  549. org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:610)
  550. Thread 22 (IPC Server Responder):
  551. State: RUNNABLE
  552. Blocked count: 6533
  553. Waited count: 6452
  554. Stack:
  555. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  556. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  557. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  558. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  559. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  560. org.apache.hadoop.hbase.ipc.HBaseServer$Responder.doRunLoop(HBaseServer.java:799)
  561. org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:782)
  562. Thread 48 (Timer-0):
  563. State: TIMED_WAITING
  564. Blocked count: 1
  565. Waited count: 19351
  566. Stack:
  567. java.lang.Object.wait(Native Method)
  568. java.util.TimerThread.mainLoop(Timer.java:509)
  569. java.util.TimerThread.run(Timer.java:462)
  570. Thread 47 (1136341612@qtp-941866449-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030):
  571. State: RUNNABLE
  572. Blocked count: 1
  573. Waited count: 1
  574. Stack:
  575. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  576. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  577. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  578. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  579. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  580. org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
  581. org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
  582. org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
  583. org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
  584. org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
  585. Thread 32 (regionserver60020.leaseChecker):
  586. State: TIMED_WAITING
  587. Blocked count: 0
  588. Waited count: 4559366
  589. Stack:
  590. sun.misc.Unsafe.park(Native Method)
  591. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  592. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
  593. java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
  594. org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:83)
  595. java.lang.Thread.run(Thread.java:619)
  596. Thread 31 (regionserver60020.compactionChecker):
  597. State: TIMED_WAITING
  598. Blocked count: 75
  599. Waited count: 60
  600. Stack:
  601. java.lang.Object.wait(Native Method)
  602. org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
  603. org.apache.hadoop.hbase.Chore.run(Chore.java:75)
  604. java.lang.Thread.run(Thread.java:619)
  605. Thread 30 (regionserver60020.cacheFlusher):
  606. State: TIMED_WAITING
  607. Blocked count: 40438
  608. Waited count: 70695
  609. Stack:
  610. sun.misc.Unsafe.park(Native Method)
  611. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  612. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
  613. java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
  614. java.util.concurrent.DelayQueue.poll(DelayQueue.java:39)
  615. org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:218)
  616. java.lang.Thread.run(Thread.java:619)
  617. Thread 38 (regionserver60020.logRoller):
  618. State: TIMED_WAITING
  619. Blocked count: 1226
  620. Waited count: 59826
  621. Stack:
  622. java.lang.Object.wait(Native Method)
  623. org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:76)
  624. java.lang.Thread.run(Thread.java:619)
  625. Thread 44 (Timer thread for monitoring jvm):
  626. State: TIMED_WAITING
  627. Blocked count: 4
  628. Waited count: 58050
  629. Stack:
  630. java.lang.Object.wait(Native Method)
  631. java.util.TimerThread.mainLoop(Timer.java:509)
  632. java.util.TimerThread.run(Timer.java:462)
  633. Thread 43 (Timer thread for monitoring hbase):
  634. State: TIMED_WAITING
  635. Blocked count: 3
  636. Waited count: 58050
  637. Stack:
  638. java.lang.Object.wait(Native Method)
  639. java.util.TimerThread.mainLoop(Timer.java:509)
  640. java.util.TimerThread.run(Timer.java:462)
  641. Thread 42 (regionserver60020.decayingSampleTick.1):
  642. State: TIMED_WAITING
  643. Blocked count: 0
  644. Waited count: 1160870
  645. Stack:
  646. sun.misc.Unsafe.park(Native Method)
  647. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  648. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
  649. java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
  650. java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
  651. java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
  652. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  653. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  654. java.lang.Thread.run(Thread.java:619)
  655. Thread 41 (regionserver60020.logSyncer):
  656. State: TIMED_WAITING
  657. Blocked count: 2698
  658. Waited count: 582724
  659. Stack:
  660. java.lang.Thread.sleep(Native Method)
  661. org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1193)
  662. java.lang.Thread.run(Thread.java:619)
  663. Thread 40 (LeaseChecker):
  664. State: TIMED_WAITING
  665. Blocked count: 19346
  666. Waited count: 618946
  667. Stack:
  668. java.lang.Thread.sleep(Native Method)
  669. org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1302)
  670. java.lang.Thread.run(Thread.java:619)
  671. Thread 37 (DestroyJavaVM):
  672. State: RUNNABLE
  673. Blocked count: 0
  674. Waited count: 0
  675. Stack:
  676. Thread 33 (IPC Client (47) connection to hmaster101.mentacapital.local./10.10.10.156:60000 from hbase):
  677. State: TIMED_WAITING
  678. Blocked count: 193043
  679. Waited count: 193043
  680. Stack:
  681. java.lang.Object.wait(Native Method)
  682. org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:459)
  683. org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:504)
  684. Thread 29 (regionserver60020-EventThread):
  685. State: WAITING
  686. Blocked count: 4
  687. Waited count: 3
  688. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3003e926
  689. Stack:
  690. sun.misc.Unsafe.park(Native Method)
  691. java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  692. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)
  693. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)
  694. org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:493)
  695. Thread 28 (regionserver60020-SendThread(hmaster101.mentacapital.local.:2181)):
  696. State: RUNNABLE
  697. Blocked count: 240
  698. Waited count: 0
  699. Stack:
  700. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  701. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  702. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  703. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  704. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  705. org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:274)
  706. org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1035)
  707. Thread 26 (regionserver60020):
  708. State: TIMED_WAITING
  709. Blocked count: 193160
  710. Waited count: 386109
  711. Stack:
  712. java.lang.Object.wait(Native Method)
  713. org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
  714. org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:55)
  715. org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:714)
  716. java.lang.Thread.run(Thread.java:619)
  717. Thread 25 (LRU Statistics #0):
  718. State: TIMED_WAITING
  719. Blocked count: 0
  720. Waited count: 1942
  721. Stack:
  722. sun.misc.Unsafe.park(Native Method)
  723. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  724. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)
  725. java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
  726. java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)
  727. java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)
  728. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  729. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  730. java.lang.Thread.run(Thread.java:619)
  731. Thread 23 (main.LruBlockCache.EvictionThread):
  732. State: WAITING
  733. Blocked count: 29
  734. Waited count: 30
  735. Waiting on org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread@6267fe80
  736. Stack:
  737. java.lang.Object.wait(Native Method)
  738. java.lang.Object.wait(Object.java:485)
  739. org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:594)
  740. java.lang.Thread.run(Thread.java:619)
  741. Thread 21 (Timer thread for monitoring rpc):
  742. State: TIMED_WAITING
  743. Blocked count: 1
  744. Waited count: 58051
  745. Stack:
  746. java.lang.Object.wait(Native Method)
  747. java.util.TimerThread.mainLoop(Timer.java:509)
  748. java.util.TimerThread.run(Timer.java:462)
  749. Thread 20 (IPC Reader 9 on port 60020):
  750. State: RUNNABLE
  751. Blocked count: 1371
  752. Waited count: 1359
  753. Stack:
  754. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  755. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  756. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  757. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  758. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  759. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  760. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  761. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  762. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  763. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  764. java.lang.Thread.run(Thread.java:619)
  765. Thread 19 (IPC Reader 8 on port 60020):
  766. State: RUNNABLE
  767. Blocked count: 1391
  768. Waited count: 1359
  769. Stack:
  770. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  771. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  772. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  773. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  774. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  775. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  776. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  777. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  778. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  779. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  780. java.lang.Thread.run(Thread.java:619)
  781. Thread 18 (IPC Reader 7 on port 60020):
  782. State: RUNNABLE
  783. Blocked count: 1371
  784. Waited count: 1359
  785. Stack:
  786. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  787. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  788. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  789. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  790. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  791. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  792. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  793. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  794. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  795. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  796. java.lang.Thread.run(Thread.java:619)
  797. Thread 17 (IPC Reader 6 on port 60020):
  798. State: RUNNABLE
  799. Blocked count: 1367
  800. Waited count: 1359
  801. Stack:
  802. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  803. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  804. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  805. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  806. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  807. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  808. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  809. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  810. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  811. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  812. java.lang.Thread.run(Thread.java:619)
  813. Thread 16 (IPC Reader 5 on port 60020):
  814. State: RUNNABLE
  815. Blocked count: 1359
  816. Waited count: 1359
  817. Stack:
  818. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  819. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  820. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  821. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  822. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  823. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  824. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  825. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  826. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  827. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  828. java.lang.Thread.run(Thread.java:619)
  829. Thread 15 (IPC Reader 4 on port 60020):
  830. State: RUNNABLE
  831. Blocked count: 1361
  832. Waited count: 1359
  833. Stack:
  834. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  835. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  836. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  837. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  838. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  839. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  840. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  841. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  842. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  843. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  844. java.lang.Thread.run(Thread.java:619)
  845. Thread 14 (IPC Reader 3 on port 60020):
  846. State: RUNNABLE
  847. Blocked count: 1362
  848. Waited count: 1360
  849. Stack:
  850. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  851. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  852. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  853. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  854. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  855. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  856. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  857. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  858. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  859. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  860. java.lang.Thread.run(Thread.java:619)
  861. Thread 13 (IPC Reader 2 on port 60020):
  862. State: RUNNABLE
  863. Blocked count: 1361
  864. Waited count: 1360
  865. Stack:
  866. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  867. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  868. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  869. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  870. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  871. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  872. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  873. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  874. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  875. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  876. java.lang.Thread.run(Thread.java:619)
  877. Thread 12 (IPC Reader 1 on port 60020):
  878. State: RUNNABLE
  879. Blocked count: 1362
  880. Waited count: 1360
  881. Stack:
  882. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  883. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  884. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  885. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  886. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  887. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  888. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  889. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  890. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  891. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  892. java.lang.Thread.run(Thread.java:619)
  893. Thread 11 (IPC Reader 0 on port 60020):
  894. State: RUNNABLE
  895. Blocked count: 1368
  896. Waited count: 1359
  897. Stack:
  898. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  899. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  900. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  901. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  902. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  903. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  904. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:502)
  905. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:488)
  906. java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  907. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  908. java.lang.Thread.run(Thread.java:619)
  909. Thread 5 (Signal Dispatcher):
  910. State: RUNNABLE
  911. Blocked count: 0
  912. Waited count: 0
  913. Stack:
  914. Thread 3 (Finalizer):
  915. State: WAITING
  916. Blocked count: 5664
  917. Waited count: 5626
  918. Waiting on java.lang.ref.ReferenceQueue$Lock@7f81497d
  919. Stack:
  920. java.lang.Object.wait(Native Method)
  921. java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
  922. java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
  923. java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
  924. Thread 2 (Reference Handler):
  925. State: WAITING
  926. Blocked count: 7586
  927. Waited count: 7517
  928. Waiting on java.lang.ref.Reference$Lock@22a010ba
  929. Stack:
  930. java.lang.Object.wait(Native Method)
  931. java.lang.Object.wait(Object.java:485)
  932. java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!