API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Oct 22nd, 2013
157
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 96.32 KB | None | 0 0
raw download clone embed print report
  1. Process Thread Dump:
  2. 88 active threads
  3. Thread 135 (IPC Client (973282215) connection to hpcn13a188/172.16.2.60:8020 from hbase):
  4. State: TIMED_WAITING
  5. Blocked count: 0
  6. Waited count: 1
  7. Stack:
  8. java.lang.Object.wait(Native Method)
  9. org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:802)
  10. org.apache.hadoop.ipc.Client$Connection.run(Client.java:845)
  11. Thread 124 (1625917720@qtp-878095664-3):
  12. State: RUNNABLE
  13. Blocked count: 9
  14. Waited count: 11
  15. Stack:
  16. sun.management.ThreadImpl.getThreadInfo1(Native Method)
  17. sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:174)
  18. sun.management.ThreadImpl.getThreadInfo(ThreadImpl.java:139)
  19. org.apache.hadoop.util.ReflectionUtils.printThreadInfo(ReflectionUtils.java:165)
  20. org.apache.hadoop.http.HttpServer$StackServlet.doGet(HttpServer.java:937)
  21. javax.servlet.http.HttpServlet.service(HttpServlet.java:734)
  22. javax.servlet.http.HttpServlet.service(HttpServlet.java:847)
  23. org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
  24. org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
  25. org.apache.hadoop.http.lib.StaticUserWebFilter$StaticUserFilter.doFilter(StaticUserWebFilter.java:109)
  26. org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
  27. org.apache.hadoop.http.HttpServer$QuotingInputFilter.doFilter(HttpServer.java:1068)
  28. org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
  29. org.apache.hadoop.http.NoCacheFilter.doFilter(NoCacheFilter.java:45)
  30. org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
  31. org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
  32. org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
  33. org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
  34. org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:766)
  35. org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
  36. Thread 105 (PostOpenDeployTasks:d119becda37daab02cab27fed93ff99f-EventThread):
  37. State: WAITING
  38. Blocked count: 0
  39. Waited count: 3
  40. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@4cae1343
  41. Stack:
  42. sun.misc.Unsafe.park(Native Method)
  43. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  44. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  45. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  46. org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
  47. Thread 104 (PostOpenDeployTasks:d119becda37daab02cab27fed93ff99f-SendThread(hpcn13a187:2181)):
  48. State: RUNNABLE
  49. Blocked count: 0
  50. Waited count: 0
  51. Stack:
  52. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  53. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  54. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  55. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  56. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  57. org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
  58. org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
  59. Thread 101 (RS_OPEN_REGION-hpcn13a026,60020,1382440301730-0):
  60. State: WAITING
  61. Blocked count: 12
  62. Waited count: 21
  63. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@3872bb09
  64. Stack:
  65. sun.misc.Unsafe.park(Native Method)
  66. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  67. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  68. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  69. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
  70. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
  71. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  72. java.lang.Thread.run(Thread.java:724)
  73. Thread 100 (org.apache.hadoop.hdfs.PeerCache@1cde878c):
  74. State: TIMED_WAITING
  75. Blocked count: 0
  76. Waited count: 7
  77. Stack:
  78. java.lang.Thread.sleep(Native Method)
  79. org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:252)
  80. org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:39)
  81. org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:135)
  82. java.lang.Thread.run(Thread.java:724)
  83. Thread 94 (SplitLogWorker-hpcn13a026,60020,1382440301730):
  84. State: WAITING
  85. Blocked count: 199
  86. Waited count: 198
  87. Waiting on java.lang.Object@5137cd80
  88. Stack:
  89. java.lang.Object.wait(Native Method)
  90. java.lang.Object.wait(Object.java:503)
  91. org.apache.hadoop.hbase.regionserver.SplitLogWorker.taskLoop(SplitLogWorker.java:222)
  92. org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:182)
  93. java.lang.Thread.run(Thread.java:724)
  94. Thread 93 (REPL IPC Server handler 2 on 60020):
  95. State: WAITING
  96. Blocked count: 0
  97. Waited count: 1
  98. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6dc65de1
  99. Stack:
  100. sun.misc.Unsafe.park(Native Method)
  101. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  102. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  103. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  104. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  105. Thread 92 (REPL IPC Server handler 1 on 60020):
  106. State: WAITING
  107. Blocked count: 0
  108. Waited count: 1
  109. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6dc65de1
  110. Stack:
  111. sun.misc.Unsafe.park(Native Method)
  112. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  113. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  114. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  115. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  116. Thread 91 (REPL IPC Server handler 0 on 60020):
  117. State: WAITING
  118. Blocked count: 0
  119. Waited count: 1
  120. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@6dc65de1
  121. Stack:
  122. sun.misc.Unsafe.park(Native Method)
  123. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  124. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  125. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  126. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  127. Thread 90 (PRI IPC Server handler 9 on 60020):
  128. State: WAITING
  129. Blocked count: 0
  130. Waited count: 6
  131. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  132. Stack:
  133. sun.misc.Unsafe.park(Native Method)
  134. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  135. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  136. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  137. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  138. Thread 89 (PRI IPC Server handler 8 on 60020):
  139. State: WAITING
  140. Blocked count: 5
  141. Waited count: 14
  142. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  143. Stack:
  144. sun.misc.Unsafe.park(Native Method)
  145. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  146. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  147. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  148. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  149. Thread 88 (PRI IPC Server handler 7 on 60020):
  150. State: WAITING
  151. Blocked count: 0
  152. Waited count: 6
  153. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  154. Stack:
  155. sun.misc.Unsafe.park(Native Method)
  156. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  157. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  158. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  159. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  160. Thread 87 (PRI IPC Server handler 6 on 60020):
  161. State: WAITING
  162. Blocked count: 0
  163. Waited count: 6
  164. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  165. Stack:
  166. sun.misc.Unsafe.park(Native Method)
  167. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  168. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  169. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  170. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  171. Thread 86 (PRI IPC Server handler 5 on 60020):
  172. State: WAITING
  173. Blocked count: 0
  174. Waited count: 6
  175. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  176. Stack:
  177. sun.misc.Unsafe.park(Native Method)
  178. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  179. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  180. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  181. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  182. Thread 85 (PRI IPC Server handler 4 on 60020):
  183. State: WAITING
  184. Blocked count: 0
  185. Waited count: 6
  186. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  187. Stack:
  188. sun.misc.Unsafe.park(Native Method)
  189. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  190. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  191. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  192. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  193. Thread 84 (PRI IPC Server handler 3 on 60020):
  194. State: WAITING
  195. Blocked count: 0
  196. Waited count: 6
  197. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  198. Stack:
  199. sun.misc.Unsafe.park(Native Method)
  200. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  201. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  202. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  203. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  204. Thread 83 (PRI IPC Server handler 2 on 60020):
  205. State: WAITING
  206. Blocked count: 1
  207. Waited count: 6
  208. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  209. Stack:
  210. sun.misc.Unsafe.park(Native Method)
  211. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  212. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  213. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  214. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  215. Thread 82 (PRI IPC Server handler 1 on 60020):
  216. State: WAITING
  217. Blocked count: 0
  218. Waited count: 6
  219. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  220. Stack:
  221. sun.misc.Unsafe.park(Native Method)
  222. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  223. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  224. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  225. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  226. Thread 81 (PRI IPC Server handler 0 on 60020):
  227. State: WAITING
  228. Blocked count: 0
  229. Waited count: 7
  230. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@7a3d9714
  231. Stack:
  232. sun.misc.Unsafe.park(Native Method)
  233. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  234. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  235. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  236. org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1400)
  237. Thread 80 (IPC Server handler 31 on 60020):
  238. State: RUNNABLE
  239. Blocked count: 27060
  240. Waited count: 107521
  241. Stack:
  242. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  243. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  244. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  245. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  246. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  247. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  248. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  249. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  250. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  251. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  252. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  253. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  254. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  255. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  256. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  257. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  258. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  259. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  260. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  261. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  262. Thread 79 (IPC Server handler 30 on 60020):
  263. State: RUNNABLE
  264. Blocked count: 26910
  265. Waited count: 107563
  266. Stack:
  267. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  268. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  269. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  270. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  271. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  272. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  273. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  274. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  275. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  276. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  277. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  278. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  279. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  280. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  281. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  282. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  283. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  284. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  285. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  286. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  287. Thread 78 (IPC Server handler 29 on 60020):
  288. State: RUNNABLE
  289. Blocked count: 27673
  290. Waited count: 107340
  291. Stack:
  292. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  293. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  294. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  295. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  296. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  297. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  298. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  299. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  300. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  301. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  302. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  303. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  304. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  305. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  306. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  307. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  308. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  309. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  310. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  311. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  312. Thread 77 (IPC Server handler 28 on 60020):
  313. State: RUNNABLE
  314. Blocked count: 26782
  315. Waited count: 107662
  316. Stack:
  317. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  318. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  319. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  320. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  321. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  322. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  323. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  324. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  325. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  326. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  327. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  328. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  329. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  330. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  331. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  332. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  333. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  334. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  335. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  336. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  337. Thread 76 (IPC Server handler 27 on 60020):
  338. State: RUNNABLE
  339. Blocked count: 26974
  340. Waited count: 107605
  341. Stack:
  342. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  343. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  344. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  345. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  346. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  347. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  348. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  349. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  350. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  351. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  352. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  353. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  354. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  355. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  356. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  357. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  358. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  359. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  360. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  361. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  362. Thread 75 (IPC Server handler 26 on 60020):
  363. State: RUNNABLE
  364. Blocked count: 27262
  365. Waited count: 107462
  366. Stack:
  367. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  368. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  369. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  370. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  371. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  372. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  373. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  374. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  375. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  376. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  377. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  378. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  379. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  380. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  381. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  382. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  383. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  384. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  385. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  386. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  387. Thread 74 (IPC Server handler 25 on 60020):
  388. State: RUNNABLE
  389. Blocked count: 26640
  390. Waited count: 107651
  391. Stack:
  392. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  393. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  394. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  395. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  396. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  397. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  398. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  399. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  400. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  401. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  402. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  403. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  404. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  405. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  406. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  407. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  408. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  409. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  410. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  411. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  412. Thread 73 (IPC Server handler 24 on 60020):
  413. State: RUNNABLE
  414. Blocked count: 27322
  415. Waited count: 107483
  416. Stack:
  417. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  418. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  419. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  420. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  421. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  422. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  423. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  424. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  425. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  426. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  427. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  428. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  429. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  430. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  431. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  432. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  433. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  434. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  435. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  436. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  437. Thread 72 (IPC Server handler 23 on 60020):
  438. State: RUNNABLE
  439. Blocked count: 27388
  440. Waited count: 107372
  441. Stack:
  442. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  443. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  444. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  445. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  446. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  447. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  448. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  449. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  450. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  451. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  452. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  453. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  454. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  455. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  456. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  457. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  458. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  459. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  460. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  461. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  462. Thread 71 (IPC Server handler 22 on 60020):
  463. State: RUNNABLE
  464. Blocked count: 27877
  465. Waited count: 107461
  466. Stack:
  467. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  468. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  469. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  470. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  471. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  472. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  473. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  474. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  475. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  476. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  477. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  478. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  479. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  480. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  481. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  482. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  483. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  484. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  485. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  486. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  487. Thread 70 (IPC Server handler 21 on 60020):
  488. State: RUNNABLE
  489. Blocked count: 27918
  490. Waited count: 107359
  491. Stack:
  492. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  493. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  494. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  495. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  496. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  497. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  498. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  499. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  500. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  501. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  502. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  503. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  504. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  505. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  506. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  507. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  508. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  509. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  510. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  511. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  512. Thread 69 (IPC Server handler 20 on 60020):
  513. State: RUNNABLE
  514. Blocked count: 27395
  515. Waited count: 107338
  516. Stack:
  517. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  518. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  519. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  520. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  521. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  522. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  523. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  524. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  525. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  526. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  527. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  528. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  529. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  530. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  531. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  532. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  533. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  534. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  535. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  536. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  537. Thread 68 (IPC Server handler 19 on 60020):
  538. State: RUNNABLE
  539. Blocked count: 27070
  540. Waited count: 107584
  541. Stack:
  542. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  543. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  544. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  545. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  546. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  547. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  548. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  549. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  550. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  551. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  552. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  553. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  554. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  555. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  556. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  557. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  558. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  559. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  560. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  561. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  562. Thread 67 (IPC Server handler 18 on 60020):
  563. State: RUNNABLE
  564. Blocked count: 26875
  565. Waited count: 107742
  566. Stack:
  567. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  568. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  569. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  570. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  571. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  572. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  573. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  574. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  575. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  576. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  577. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  578. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  579. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  580. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  581. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  582. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  583. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  584. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  585. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  586. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  587. Thread 66 (IPC Server handler 17 on 60020):
  588. State: RUNNABLE
  589. Blocked count: 27048
  590. Waited count: 107522
  591. Stack:
  592. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  593. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  594. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  595. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  596. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  597. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  598. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  599. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  600. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  601. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  602. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  603. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  604. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  605. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  606. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  607. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  608. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  609. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  610. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  611. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  612. Thread 65 (IPC Server handler 16 on 60020):
  613. State: RUNNABLE
  614. Blocked count: 26704
  615. Waited count: 107681
  616. Stack:
  617. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  618. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  619. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  620. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  621. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  622. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  623. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  624. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  625. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  626. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  627. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  628. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  629. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  630. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  631. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  632. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  633. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  634. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  635. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  636. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  637. Thread 64 (IPC Server handler 15 on 60020):
  638. State: RUNNABLE
  639. Blocked count: 26664
  640. Waited count: 107520
  641. Stack:
  642. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  643. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  644. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  645. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  646. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  647. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  648. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  649. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  650. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  651. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  652. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  653. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  654. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  655. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  656. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  657. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  658. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  659. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  660. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  661. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  662. Thread 63 (IPC Server handler 14 on 60020):
  663. State: RUNNABLE
  664. Blocked count: 27247
  665. Waited count: 107645
  666. Stack:
  667. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  668. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  669. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  670. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  671. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  672. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  673. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  674. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  675. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  676. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  677. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  678. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  679. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  680. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  681. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  682. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  683. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  684. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  685. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  686. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  687. Thread 62 (IPC Server handler 13 on 60020):
  688. State: RUNNABLE
  689. Blocked count: 27198
  690. Waited count: 107370
  691. Stack:
  692. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  693. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  694. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  695. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  696. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  697. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  698. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  699. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  700. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  701. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  702. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  703. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  704. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  705. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  706. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  707. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  708. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  709. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  710. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  711. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  712. Thread 61 (IPC Server handler 12 on 60020):
  713. State: RUNNABLE
  714. Blocked count: 27130
  715. Waited count: 107491
  716. Stack:
  717. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  718. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  719. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  720. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  721. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  722. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  723. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  724. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  725. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  726. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  727. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  728. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  729. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  730. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  731. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  732. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  733. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  734. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  735. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  736. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  737. Thread 60 (IPC Server handler 11 on 60020):
  738. State: RUNNABLE
  739. Blocked count: 27714
  740. Waited count: 107570
  741. Stack:
  742. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  743. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  744. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  745. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  746. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  747. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  748. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  749. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  750. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  751. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  752. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  753. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  754. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  755. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  756. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  757. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  758. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  759. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  760. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  761. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  762. Thread 59 (IPC Server handler 10 on 60020):
  763. State: RUNNABLE
  764. Blocked count: 26433
  765. Waited count: 107705
  766. Stack:
  767. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  768. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  769. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  770. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  771. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  772. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  773. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  774. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  775. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  776. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  777. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  778. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  779. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  780. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  781. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  782. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  783. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  784. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  785. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  786. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  787. Thread 58 (IPC Server handler 9 on 60020):
  788. State: RUNNABLE
  789. Blocked count: 27667
  790. Waited count: 107386
  791. Stack:
  792. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  793. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  794. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  795. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  796. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  797. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  798. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  799. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  800. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  801. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  802. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  803. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  804. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  805. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  806. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  807. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  808. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  809. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  810. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  811. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  812. Thread 57 (IPC Server handler 8 on 60020):
  813. State: RUNNABLE
  814. Blocked count: 27254
  815. Waited count: 107579
  816. Stack:
  817. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  818. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  819. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  820. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  821. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  822. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  823. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  824. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  825. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  826. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  827. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  828. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  829. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  830. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  831. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  832. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  833. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  834. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  835. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  836. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  837. Thread 56 (IPC Server handler 7 on 60020):
  838. State: RUNNABLE
  839. Blocked count: 26978
  840. Waited count: 107629
  841. Stack:
  842. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1354)
  843. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  844. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  845. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  846. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  847. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  848. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  849. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  850. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  851. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  852. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  853. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  854. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  855. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  856. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  857. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  858. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  859. org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4725)
  860. org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:4699)
  861. org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:2072)
  862. Thread 55 (IPC Server handler 6 on 60020):
  863. State: RUNNABLE
  864. Blocked count: 27603
  865. Waited count: 107479
  866. Stack:
  867. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  868. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  869. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  870. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  871. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  872. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  873. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  874. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  875. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  876. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  877. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  878. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  879. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  880. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  881. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  882. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  883. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  884. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  885. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  886. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  887. Thread 54 (IPC Server handler 5 on 60020):
  888. State: RUNNABLE
  889. Blocked count: 27792
  890. Waited count: 107446
  891. Stack:
  892. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  893. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  894. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  895. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  896. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  897. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  898. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  899. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  900. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  901. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  902. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  903. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  904. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  905. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  906. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  907. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  908. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  909. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  910. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  911. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  912. Thread 53 (IPC Server handler 4 on 60020):
  913. State: RUNNABLE
  914. Blocked count: 27703
  915. Waited count: 107270
  916. Stack:
  917. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  918. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  919. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  920. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  921. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  922. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  923. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  924. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  925. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  926. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  927. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  928. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  929. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  930. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  931. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  932. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  933. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  934. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  935. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  936. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  937. Thread 52 (IPC Server handler 3 on 60020):
  938. State: RUNNABLE
  939. Blocked count: 26319
  940. Waited count: 107444
  941. Stack:
  942. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  943. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  944. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  945. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  946. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  947. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  948. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  949. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  950. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  951. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  952. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  953. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  954. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  955. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  956. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  957. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  958. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  959. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  960. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  961. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  962. Thread 51 (IPC Server handler 2 on 60020):
  963. State: RUNNABLE
  964. Blocked count: 27045
  965. Waited count: 107544
  966. Stack:
  967. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  968. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  969. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  970. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  971. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  972. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  973. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  974. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  975. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  976. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  977. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  978. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  979. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  980. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  981. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  982. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  983. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  984. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  985. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  986. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  987. Thread 50 (IPC Server handler 1 on 60020):
  988. State: RUNNABLE
  989. Blocked count: 27726
  990. Waited count: 107545
  991. Stack:
  992. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  993. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  994. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  995. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  996. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  997. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  998. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  999. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  1000. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  1001. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  1002. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  1003. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:805)
  1004. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  1005. org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:321)
  1006. org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:277)
  1007. org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:543)
  1008. org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:411)
  1009. org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:143)
  1010. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:3835)
  1011. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:3902)
  1012. Thread 49 (IPC Server handler 0 on 60020):
  1013. State: RUNNABLE
  1014. Blocked count: 26705
  1015. Waited count: 107597
  1016. Stack:
  1017. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1535)
  1018. org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1523)
  1019. java.util.concurrent.ConcurrentSkipListMap$ComparableUsingComparator.compareTo(ConcurrentSkipListMap.java:647)
  1020. java.util.concurrent.ConcurrentSkipListMap.findNear(ConcurrentSkipListMap.java:1346)
  1021. java.util.concurrent.ConcurrentSkipListMap$SubMap.loNode(ConcurrentSkipListMap.java:2571)
  1022. java.util.concurrent.ConcurrentSkipListMap$SubMap.access$300(ConcurrentSkipListMap.java:2474)
  1023. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapIter.<init>(ConcurrentSkipListMap.java:3010)
  1024. java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapValueIterator.<init>(ConcurrentSkipListMap.java:3080)
  1025. java.util.concurrent.ConcurrentSkipListMap$SubMap.valueIterator(ConcurrentSkipListMap.java:2990)
  1026. java.util.concurrent.ConcurrentSkipListMap$Values.iterator(ConcurrentSkipListMap.java:2390)
  1027. org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet.iterator(KeyValueSkipListSet.java:87)
  1028. org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.seek(MemStore.java:761)
  1029. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:54)
  1030. org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.requestSeek(NonLazyKeyValueScanner.java:38)
  1031. org.apache.hadoop.hbase.regionserver.StoreScanner.<init>(StoreScanner.java:127)
  1032. org.apache.hadoop.hbase.regionserver.Store.getScanner(Store.java:2135)
  1033. org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.<init>(HRegion.java:3714)
  1034. org.apache.hadoop.hbase.regionserver.HRegion.instantiateRegionScanner(HRegion.java:1761)
  1035. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1753)
  1036. org.apache.hadoop.hbase.regionserver.HRegion.getScanner(HRegion.java:1730)
  1037. Thread 11 (IPC Server listener on 60020):
  1038. State: RUNNABLE
  1039. Blocked count: 3
  1040. Waited count: 0
  1041. Stack:
  1042. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1043. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1044. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1045. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1046. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1047. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1048. org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:638)
  1049. Thread 23 (IPC Server Responder):
  1050. State: RUNNABLE
  1051. Blocked count: 2
  1052. Waited count: 2
  1053. Stack:
  1054. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1055. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1056. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1057. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1058. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1059. org.apache.hadoop.hbase.ipc.HBaseServer$Responder.doRunLoop(HBaseServer.java:827)
  1060. org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:810)
  1061. Thread 48 (Timer-0):
  1062. State: TIMED_WAITING
  1063. Blocked count: 0
  1064. Waited count: 30
  1065. Stack:
  1066. java.lang.Object.wait(Native Method)
  1067. java.util.TimerThread.mainLoop(Timer.java:552)
  1068. java.util.TimerThread.run(Timer.java:505)
  1069. Thread 47 (1101082627@qtp-878095664-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030):
  1070. State: RUNNABLE
  1071. Blocked count: 3
  1072. Waited count: 1
  1073. Stack:
  1074. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1075. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1076. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1077. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1078. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1079. org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:511)
  1080. org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:193)
  1081. org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
  1082. org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
  1083. org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
  1084. Thread 34 (regionserver60020.leaseChecker):
  1085. State: TIMED_WAITING
  1086. Blocked count: 0
  1087. Waited count: 88
  1088. Stack:
  1089. sun.misc.Unsafe.park(Native Method)
  1090. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
  1091. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
  1092. java.util.concurrent.DelayQueue.poll(DelayQueue.java:256)
  1093. org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:83)
  1094. java.lang.Thread.run(Thread.java:724)
  1095. Thread 33 (regionserver60020.compactionChecker):
  1096. State: TIMED_WAITING
  1097. Blocked count: 0
  1098. Waited count: 1
  1099. Stack:
  1100. java.lang.Object.wait(Native Method)
  1101. org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
  1102. org.apache.hadoop.hbase.Chore.run(Chore.java:75)
  1103. java.lang.Thread.run(Thread.java:724)
  1104. Thread 32 (regionserver60020.cacheFlusher):
  1105. State: TIMED_WAITING
  1106. Blocked count: 0
  1107. Waited count: 88
  1108. Stack:
  1109. sun.misc.Unsafe.park(Native Method)
  1110. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
  1111. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
  1112. java.util.concurrent.DelayQueue.poll(DelayQueue.java:256)
  1113. java.util.concurrent.DelayQueue.poll(DelayQueue.java:68)
  1114. org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:225)
  1115. java.lang.Thread.run(Thread.java:724)
  1116. Thread 39 (regionserver60020.logRoller):
  1117. State: TIMED_WAITING
  1118. Blocked count: 0
  1119. Waited count: 88
  1120. Stack:
  1121. java.lang.Object.wait(Native Method)
  1122. org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:76)
  1123. java.lang.Thread.run(Thread.java:724)
  1124. Thread 44 (Timer thread for monitoring jvm):
  1125. State: TIMED_WAITING
  1126. Blocked count: 0
  1127. Waited count: 15
  1128. Stack:
  1129. java.lang.Object.wait(Native Method)
  1130. java.util.TimerThread.mainLoop(Timer.java:552)
  1131. java.util.TimerThread.run(Timer.java:505)
  1132. Thread 43 (Timer thread for monitoring hbase):
  1133. State: TIMED_WAITING
  1134. Blocked count: 0
  1135. Waited count: 15
  1136. Stack:
  1137. java.lang.Object.wait(Native Method)
  1138. java.util.TimerThread.mainLoop(Timer.java:552)
  1139. java.util.TimerThread.run(Timer.java:505)
  1140. Thread 42 (regionserver60020.logSyncer):
  1141. State: TIMED_WAITING
  1142. Blocked count: 0
  1143. Waited count: 873
  1144. Stack:
  1145. java.lang.Object.wait(Native Method)
  1146. org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:1253)
  1147. java.lang.Thread.run(Thread.java:724)
  1148. Thread 41 (LeaseRenewer:hbase@hpcn13a188:8020):
  1149. State: TIMED_WAITING
  1150. Blocked count: 29
  1151. Waited count: 929
  1152. Stack:
  1153. java.lang.Thread.sleep(Native Method)
  1154. org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:438)
  1155. org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71)
  1156. org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:298)
  1157. java.lang.Thread.run(Thread.java:724)
  1158. Thread 40 (Thread-13):
  1159. State: TIMED_WAITING
  1160. Blocked count: 0
  1161. Waited count: 873
  1162. Stack:
  1163. java.lang.Object.wait(Native Method)
  1164. org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:465)
  1165. Thread 38 (IPC Parameter Sending Thread #0):
  1166. State: TIMED_WAITING
  1167. Blocked count: 0
  1168. Waited count: 48
  1169. Stack:
  1170. sun.misc.Unsafe.park(Native Method)
  1171. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
  1172. java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
  1173. java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
  1174. java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
  1175. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
  1176. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
  1177. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1178. java.lang.Thread.run(Thread.java:724)
  1179. Thread 36 (IPC Client (973282215) connection to hpcn13a188/172.16.2.60:60000 from hbase):
  1180. State: TIMED_WAITING
  1181. Blocked count: 294
  1182. Waited count: 294
  1183. Stack:
  1184. java.lang.Object.wait(Native Method)
  1185. org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:540)
  1186. org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:585)
  1187. Thread 29 (regionserver60020-EventThread):
  1188. State: WAITING
  1189. Blocked count: 11
  1190. Waited count: 73
  1191. Waiting on java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject@2b475e55
  1192. Stack:
  1193. sun.misc.Unsafe.park(Native Method)
  1194. java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
  1195. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
  1196. java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1197. org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
  1198. Thread 28 (regionserver60020-SendThread(hpcn13a187:2181)):
  1199. State: RUNNABLE
  1200. Blocked count: 0
  1201. Waited count: 0
  1202. Stack:
  1203. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1204. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1205. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1206. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1207. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1208. org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
  1209. org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
  1210. Thread 27 (regionserver60020):
  1211. State: TIMED_WAITING
  1212. Blocked count: 317
  1213. Waited count: 628
  1214. Stack:
  1215. java.lang.Object.wait(Native Method)
  1216. org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
  1217. org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:55)
  1218. org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:773)
  1219. java.lang.Thread.run(Thread.java:724)
  1220. Thread 26 (LRU Statistics #0):
  1221. State: TIMED_WAITING
  1222. Blocked count: 0
  1223. Waited count: 3
  1224. Stack:
  1225. sun.misc.Unsafe.park(Native Method)
  1226. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
  1227. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
  1228. java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1090)
  1229. java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
  1230. java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
  1231. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
  1232. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1233. java.lang.Thread.run(Thread.java:724)
  1234. Thread 24 (main.LruBlockCache.EvictionThread):
  1235. State: WAITING
  1236. Blocked count: 0
  1237. Waited count: 1
  1238. Waiting on org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread@1256b347
  1239. Stack:
  1240. java.lang.Object.wait(Native Method)
  1241. java.lang.Object.wait(Object.java:503)
  1242. org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:631)
  1243. java.lang.Thread.run(Thread.java:724)
  1244. Thread 22 (Timer thread for monitoring rpc):
  1245. State: TIMED_WAITING
  1246. Blocked count: 0
  1247. Waited count: 15
  1248. Stack:
  1249. java.lang.Object.wait(Native Method)
  1250. java.util.TimerThread.mainLoop(Timer.java:552)
  1251. java.util.TimerThread.run(Timer.java:505)
  1252. Thread 21 (IPC Reader 9 on port 60020):
  1253. State: RUNNABLE
  1254. Blocked count: 0
  1255. Waited count: 0
  1256. Stack:
  1257. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1258. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1259. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1260. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1261. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1262. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1263. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1264. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1265. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1266. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1267. java.lang.Thread.run(Thread.java:724)
  1268. Thread 20 (IPC Reader 8 on port 60020):
  1269. State: RUNNABLE
  1270. Blocked count: 0
  1271. Waited count: 0
  1272. Stack:
  1273. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1274. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1275. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1276. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1277. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1278. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1279. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1280. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1281. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1282. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1283. java.lang.Thread.run(Thread.java:724)
  1284. Thread 19 (IPC Reader 7 on port 60020):
  1285. State: RUNNABLE
  1286. Blocked count: 0
  1287. Waited count: 0
  1288. Stack:
  1289. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1290. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1291. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1292. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1293. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1294. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1295. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1296. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1297. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1298. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1299. java.lang.Thread.run(Thread.java:724)
  1300. Thread 18 (IPC Reader 6 on port 60020):
  1301. State: RUNNABLE
  1302. Blocked count: 0
  1303. Waited count: 0
  1304. Stack:
  1305. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1306. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1307. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1308. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1309. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1310. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1311. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1312. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1313. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1314. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1315. java.lang.Thread.run(Thread.java:724)
  1316. Thread 17 (IPC Reader 5 on port 60020):
  1317. State: RUNNABLE
  1318. Blocked count: 0
  1319. Waited count: 0
  1320. Stack:
  1321. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1322. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1323. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1324. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1325. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1326. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1327. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1328. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1329. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1330. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1331. java.lang.Thread.run(Thread.java:724)
  1332. Thread 16 (IPC Reader 4 on port 60020):
  1333. State: RUNNABLE
  1334. Blocked count: 76705
  1335. Waited count: 12234
  1336. Stack:
  1337. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1338. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1339. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1340. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1341. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1342. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1343. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1344. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1345. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1346. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1347. java.lang.Thread.run(Thread.java:724)
  1348. Thread 15 (IPC Reader 3 on port 60020):
  1349. State: RUNNABLE
  1350. Blocked count: 1
  1351. Waited count: 1
  1352. Stack:
  1353. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1354. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1355. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1356. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1357. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1358. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1359. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1360. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1361. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1362. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1363. java.lang.Thread.run(Thread.java:724)
  1364. Thread 14 (IPC Reader 2 on port 60020):
  1365. State: RUNNABLE
  1366. Blocked count: 1
  1367. Waited count: 1
  1368. Stack:
  1369. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1370. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1371. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1372. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1373. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1374. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1375. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1376. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1377. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1378. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1379. java.lang.Thread.run(Thread.java:724)
  1380. Thread 13 (IPC Reader 1 on port 60020):
  1381. State: RUNNABLE
  1382. Blocked count: 1
  1383. Waited count: 1
  1384. Stack:
  1385. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1386. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1387. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1388. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1389. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1390. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1391. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1392. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1393. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1394. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1395. java.lang.Thread.run(Thread.java:724)
  1396. Thread 12 (IPC Reader 0 on port 60020):
  1397. State: RUNNABLE
  1398. Blocked count: 0
  1399. Waited count: 0
  1400. Stack:
  1401. sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1402. sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:257)
  1403. sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1404. sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
  1405. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
  1406. sun.nio.ch.SelectorImpl.select(SelectorImpl.java:102)
  1407. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.doRunLoop(HBaseServer.java:530)
  1408. org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:516)
  1409. java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  1410. java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  1411. java.lang.Thread.run(Thread.java:724)
  1412. Thread 9 (event filterer):
  1413. State: TIMED_WAITING
  1414. Blocked count: 0
  1415. Waited count: 542
  1416. Stack:
  1417. sun.misc.Unsafe.park(Native Method)
  1418. java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
  1419. java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2082)
  1420. java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
  1421. com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.runFiltering(LogEventProcessor.java:132)
  1422. com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.access$000(LogEventProcessor.java:28)
  1423. com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor$1.run(LogEventProcessor.java:81)
  1424. Thread 5 (Signal Dispatcher):
  1425. State: RUNNABLE
  1426. Blocked count: 0
  1427. Waited count: 0
  1428. Stack:
  1429. Thread 3 (Finalizer):
  1430. State: WAITING
  1431. Blocked count: 30
  1432. Waited count: 9
  1433. Waiting on java.lang.ref.ReferenceQueue$Lock@218e025c
  1434. Stack:
  1435. java.lang.Object.wait(Native Method)
  1436. java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:135)
  1437. java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:151)
  1438. java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:189)
  1439. Thread 2 (Reference Handler):
  1440. State: WAITING
  1441. Blocked count: 17
  1442. Waited count: 15
  1443. Waiting on java.lang.ref.Reference$Lock@723cf739
  1444. Stack:
  1445. java.lang.Object.wait(Native Method)
  1446. java.lang.Object.wait(Object.java:503)
  1447. java.lang.ref.Reference$ReferenceHandler.run(Reference.java:133)
  1448. Thread 1 (main):
  1449. State: WAITING
  1450. Blocked count: 2
  1451. Waited count: 1
  1452. Waiting on java.lang.Thread@8af9c7
  1453. Stack:
  1454. java.lang.Object.wait(Native Method)
  1455. java.lang.Thread.join(Thread.java:1260)
  1456. java.lang.Thread.join(Thread.java:1334)
  1457. org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:63)
  1458. org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:84)
  1459. org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
  1460. org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:76)
  1461. org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:3907)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!