1. Full thread dump Java HotSpot(TM) 64-Bit Server VM (19.1-b02 mixed mode):
  2.  
  3. "ResponseProcessor for block blk_-7016532988952896340_1175634" daemon prio=10 tid=0x00007f924c5e9000 nid=0x2dee runnable [0x00007f91f1445000]
  4. java.lang.Thread.State: RUNNABLE
  5. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  6. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  7. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  8. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  9. - locked <0x000000060d81a998> (a sun.nio.ch.Util$2)
  10. - locked <0x000000060d81a980> (a java.util.Collections$UnmodifiableSet)
  11. - locked <0x000000059a32b6d0> (a sun.nio.ch.EPollSelectorImpl)
  12. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  13. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
  14. at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  15. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
  16. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
  17. at java.io.DataInputStream.readFully(DataInputStream.java:178)
  18. at java.io.DataInputStream.readLong(DataInputStream.java:399)
  19. at org.apache.hadoop.hdfs.protocol.DataTransferProtocol$PipelineAck.readFields(DataTransferProtocol.java:119)
  20. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$ResponseProcessor.run(DFSClient.java:2468)
  21.  
  22. "DataStreamer for file /hbase/.logs/mtag4.prod.imageshack.com,60020,1305415354243/10.103.7.4%3A60020.1305494911549 block blk_-7016532988952896340_1175634" daemon prio=10 tid=0x00007f9180010800 nid=0x2de7 in Object.wait() [0x00007f912e3e2000]
  23. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  24. at java.lang.Object.wait(Native Method)
  25. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2313)
  26. - locked <0x00000006b656a950> (a java.util.LinkedList)
  27.  
  28. "IPC Client (47) connection to /10.103.7.2:60000 from an unknown user" daemon prio=10 tid=0x00007f91a80aa800 nid=0x3639 in Object.wait() [0x00007f912f5f4000]
  29. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  30. at java.lang.Object.wait(Native Method)
  31. at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:408)
  32. - locked <0x000000058c3b55b0> (a org.apache.hadoop.hbase.ipc.HBaseClient$Connection)
  33. at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:453)
  34.  
  35. "regionserver60020.compactor-EventThread" daemon prio=10 tid=0x00007f9178037000 nid=0x12df waiting on condition [0x00007f91f1647000]
  36. java.lang.Thread.State: WAITING (parking)
  37. at sun.misc.Unsafe.park(Native Method)
  38. - parking to wait for <0x00000005bd0bee78> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  39. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  40. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  41. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  42. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:477)
  43.  
  44. "regionserver60020.compactor-SendThread(mtag7:2181)" daemon prio=10 tid=0x00007f9178030800 nid=0x12de runnable [0x00007f91f1344000]
  45. java.lang.Thread.State: RUNNABLE
  46. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  47. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  48. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  49. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  50. - locked <0x00000005bd103780> (a sun.nio.ch.Util$2)
  51. - locked <0x00000005bd105838> (a java.util.Collections$UnmodifiableSet)
  52. - locked <0x0000000544de4d20> (a sun.nio.ch.EPollSelectorImpl)
  53. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  54. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1066)
  55.  
  56. "2139641030@qtp-877230050-2" prio=10 tid=0x00007f915c013800 nid=0x5b12 in Object.wait() [0x00007f91484c3000]
  57. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  58. at java.lang.Object.wait(Native Method)
  59. at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
  60. - locked <0x0000000645a336d8> (a org.mortbay.thread.QueuedThreadPool$PoolThread)
  61.  
  62. "pool-3-thread-1" prio=10 tid=0x00007f91a8223000 nid=0x20fe waiting on condition [0x00007f91485c4000]
  63. java.lang.Thread.State: TIMED_WAITING (parking)
  64. at sun.misc.Unsafe.park(Native Method)
  65. - parking to wait for <0x0000000518a3fcf8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  66. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  67. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  68. at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
  69. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609)
  70. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602)
  71. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  72. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  73. at java.lang.Thread.run(Thread.java:662)
  74.  
  75. "LruBlockCache.EvictionThread" daemon prio=10 tid=0x00007f91a8222000 nid=0x20fd in Object.wait() [0x00007f91486c5000]
  76. java.lang.Thread.State: WAITING (on object monitor)
  77. at java.lang.Object.wait(Native Method)
  78. at java.lang.Object.wait(Object.java:485)
  79. at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:514)
  80. - locked <0x0000000518b268b8> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread)
  81.  
  82. "PRI IPC Server handler 9 on 60020" daemon prio=10 tid=0x00007f91a821f800 nid=0x20fc waiting on condition [0x00007f91487c6000]
  83. java.lang.Thread.State: WAITING (parking)
  84. at sun.misc.Unsafe.park(Native Method)
  85. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  86. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  87. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  88. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  89. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  90.  
  91. "PRI IPC Server handler 8 on 60020" daemon prio=10 tid=0x00007f91a821d800 nid=0x20fb waiting on condition [0x00007f91488c7000]
  92. java.lang.Thread.State: WAITING (parking)
  93. at sun.misc.Unsafe.park(Native Method)
  94. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  95. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  96. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  97. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  98. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  99.  
  100. "PRI IPC Server handler 7 on 60020" daemon prio=10 tid=0x00007f91a821b800 nid=0x20fa waiting on condition [0x00007f91489c8000]
  101. java.lang.Thread.State: WAITING (parking)
  102. at sun.misc.Unsafe.park(Native Method)
  103. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  104. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  105. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  106. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  107. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  108.  
  109. "PRI IPC Server handler 6 on 60020" daemon prio=10 tid=0x00007f91a8219800 nid=0x20f9 waiting on condition [0x00007f9148ac9000]
  110. java.lang.Thread.State: WAITING (parking)
  111. at sun.misc.Unsafe.park(Native Method)
  112. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  113. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  114. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  115. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  116. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  117.  
  118. "PRI IPC Server handler 5 on 60020" daemon prio=10 tid=0x00007f91a8217800 nid=0x20f8 waiting on condition [0x00007f9148bca000]
  119. java.lang.Thread.State: WAITING (parking)
  120. at sun.misc.Unsafe.park(Native Method)
  121. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  122. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  123. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  124. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  125. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  126.  
  127. "PRI IPC Server handler 4 on 60020" daemon prio=10 tid=0x00007f91a8215800 nid=0x20f7 waiting on condition [0x00007f9148ccb000]
  128. java.lang.Thread.State: WAITING (parking)
  129. at sun.misc.Unsafe.park(Native Method)
  130. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  131. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  132. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  133. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  134. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  135.  
  136. "PRI IPC Server handler 3 on 60020" daemon prio=10 tid=0x00007f91a8213800 nid=0x20f6 waiting on condition [0x00007f9148dcc000]
  137. java.lang.Thread.State: WAITING (parking)
  138. at sun.misc.Unsafe.park(Native Method)
  139. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  140. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  141. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  142. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  143. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  144.  
  145. "PRI IPC Server handler 2 on 60020" daemon prio=10 tid=0x00007f91a8211800 nid=0x20f5 waiting on condition [0x00007f9148ecd000]
  146. java.lang.Thread.State: WAITING (parking)
  147. at sun.misc.Unsafe.park(Native Method)
  148. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  149. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  150. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  151. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  152. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  153.  
  154. "PRI IPC Server handler 1 on 60020" daemon prio=10 tid=0x00007f91a820f800 nid=0x20f4 waiting on condition [0x00007f9148fce000]
  155. java.lang.Thread.State: WAITING (parking)
  156. at sun.misc.Unsafe.park(Native Method)
  157. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  158. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  159. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  160. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  161. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  162.  
  163. "PRI IPC Server handler 0 on 60020" daemon prio=10 tid=0x00007f91a820d800 nid=0x20f3 waiting on condition [0x00007f91490cf000]
  164. java.lang.Thread.State: WAITING (parking)
  165. at sun.misc.Unsafe.park(Native Method)
  166. - parking to wait for <0x0000000518883b60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  167. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  168. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  169. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  170. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  171.  
  172. "IPC Server handler 49 on 60020" daemon prio=10 tid=0x00007f91a820b800 nid=0x20f2 waiting on condition [0x00007f91491d0000]
  173. java.lang.Thread.State: WAITING (parking)
  174. at sun.misc.Unsafe.park(Native Method)
  175. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  176. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  177. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  178. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  179. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  180.  
  181. "IPC Server handler 48 on 60020" daemon prio=10 tid=0x00007f91a8209800 nid=0x20f1 waiting on condition [0x00007f91492d1000]
  182. java.lang.Thread.State: WAITING (parking)
  183. at sun.misc.Unsafe.park(Native Method)
  184. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  185. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  186. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  187. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  188. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  189.  
  190. "IPC Server handler 47 on 60020" daemon prio=10 tid=0x00007f91a8207800 nid=0x20f0 waiting on condition [0x00007f91493d2000]
  191. java.lang.Thread.State: WAITING (parking)
  192. at sun.misc.Unsafe.park(Native Method)
  193. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  194. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  195. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  196. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  197. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  198.  
  199. "IPC Server handler 46 on 60020" daemon prio=10 tid=0x00007f91a8205800 nid=0x20ef runnable [0x00007f91494d3000]
  200. java.lang.Thread.State: RUNNABLE
  201. at org.apache.hadoop.hbase.KeyValue$KVComparator.compare(KeyValue.java:1356)
  202. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:604)
  203. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:240)
  204. at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:382)
  205. - locked <0x000000050d17dd58> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  206. at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:288)
  207. - locked <0x000000050d17dd58> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  208. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:114)
  209. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.nextInternal(HRegion.java:2284)
  210. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2241)
  211. - locked <0x000000050d17dc58> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  212. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2257)
  213. - locked <0x000000050d17dc58> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  214. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2943)
  215. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2847)
  216. at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1746)
  217. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  218. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  219. at java.lang.reflect.Method.invoke(Method.java:597)
  220. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560)
  221. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027)
  222.  
  223. "IPC Server handler 45 on 60020" daemon prio=10 tid=0x00007f91a8203800 nid=0x20ee waiting on condition [0x00007f91495d4000]
  224. java.lang.Thread.State: WAITING (parking)
  225. at sun.misc.Unsafe.park(Native Method)
  226. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  227. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  228. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  229. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  230. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  231.  
  232. "IPC Server handler 44 on 60020" daemon prio=10 tid=0x00007f91a8201800 nid=0x20ed waiting on condition [0x00007f91496d5000]
  233. java.lang.Thread.State: WAITING (parking)
  234. at sun.misc.Unsafe.park(Native Method)
  235. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  236. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  237. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  238. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  239. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  240.  
  241. "IPC Server handler 43 on 60020" daemon prio=10 tid=0x00007f91a81ff800 nid=0x20ec waiting on condition [0x00007f91497d6000]
  242. java.lang.Thread.State: WAITING (parking)
  243. at sun.misc.Unsafe.park(Native Method)
  244. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  245. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  246. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  247. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  248. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  249.  
  250. "IPC Server handler 42 on 60020" daemon prio=10 tid=0x00007f91a81fd800 nid=0x20eb waiting on condition [0x00007f91498d7000]
  251. java.lang.Thread.State: WAITING (parking)
  252. at sun.misc.Unsafe.park(Native Method)
  253. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  254. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  255. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  256. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  257. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  258.  
  259. "IPC Server handler 41 on 60020" daemon prio=10 tid=0x00007f91a81fb800 nid=0x20ea waiting on condition [0x00007f91499d8000]
  260. java.lang.Thread.State: WAITING (parking)
  261. at sun.misc.Unsafe.park(Native Method)
  262. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  263. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  264. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  265. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  266. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  267.  
  268. "IPC Server handler 40 on 60020" daemon prio=10 tid=0x00007f91a81f9800 nid=0x20e9 waiting on condition [0x00007f9149ad9000]
  269. java.lang.Thread.State: WAITING (parking)
  270. at sun.misc.Unsafe.park(Native Method)
  271. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  272. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  273. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  274. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  275. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  276.  
  277. "IPC Server handler 39 on 60020" daemon prio=10 tid=0x00007f91a81f7800 nid=0x20e8 waiting on condition [0x00007f9149bda000]
  278. java.lang.Thread.State: WAITING (parking)
  279. at sun.misc.Unsafe.park(Native Method)
  280. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  281. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  282. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  283. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  284. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  285.  
  286. "IPC Server handler 38 on 60020" daemon prio=10 tid=0x00007f91a81f5800 nid=0x20e7 waiting on condition [0x00007f9149cdb000]
  287. java.lang.Thread.State: WAITING (parking)
  288. at sun.misc.Unsafe.park(Native Method)
  289. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  290. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  291. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  292. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  293. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  294.  
  295. "IPC Server handler 37 on 60020" daemon prio=10 tid=0x00007f91a81f3800 nid=0x20e6 waiting on condition [0x00007f9149ddc000]
  296. java.lang.Thread.State: WAITING (parking)
  297. at sun.misc.Unsafe.park(Native Method)
  298. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  299. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  300. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  301. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  302. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  303.  
  304. "IPC Server handler 36 on 60020" daemon prio=10 tid=0x00007f91a81f1800 nid=0x20e5 waiting on condition [0x00007f9149edd000]
  305. java.lang.Thread.State: WAITING (parking)
  306. at sun.misc.Unsafe.park(Native Method)
  307. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  308. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  309. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  310. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  311. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  312.  
  313. "IPC Server handler 35 on 60020" daemon prio=10 tid=0x00007f91a81ef800 nid=0x20e4 waiting on condition [0x00007f9149fde000]
  314. java.lang.Thread.State: WAITING (parking)
  315. at sun.misc.Unsafe.park(Native Method)
  316. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  317. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  318. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  319. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  320. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  321.  
  322. "IPC Server handler 34 on 60020" daemon prio=10 tid=0x00007f91a81ed800 nid=0x20e3 runnable [0x00007f914a0df000]
  323. java.lang.Thread.State: RUNNABLE
  324. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3113)
  325. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3108)
  326. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:74)
  327. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:62)
  328. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.getNext(MemStore.java:561)
  329. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:605)
  330. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:240)
  331. at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:382)
  332. - locked <0x000000050d8a2880> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  333. at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:288)
  334. - locked <0x000000050d8a2880> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  335. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:114)
  336. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.nextInternal(HRegion.java:2284)
  337. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2241)
  338. - locked <0x000000050d8a2780> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  339. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2257)
  340. - locked <0x000000050d8a2780> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  341. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2943)
  342. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2847)
  343. at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1746)
  344. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  345. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  346. at java.lang.reflect.Method.invoke(Method.java:597)
  347. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560)
  348. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027)
  349.  
  350. "IPC Server handler 33 on 60020" daemon prio=10 tid=0x00007f91a81eb800 nid=0x20e2 waiting on condition [0x00007f914a1e0000]
  351. java.lang.Thread.State: WAITING (parking)
  352. at sun.misc.Unsafe.park(Native Method)
  353. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  354. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  355. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  356. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  357. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  358.  
  359. "IPC Server handler 32 on 60020" daemon prio=10 tid=0x00007f91a81e9800 nid=0x20e1 waiting on condition [0x00007f914a2e1000]
  360. java.lang.Thread.State: WAITING (parking)
  361. at sun.misc.Unsafe.park(Native Method)
  362. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  363. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  364. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  365. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  366. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  367.  
  368. "IPC Server handler 31 on 60020" daemon prio=10 tid=0x00007f91a81e7800 nid=0x20e0 waiting on condition [0x00007f914a3e2000]
  369. java.lang.Thread.State: WAITING (parking)
  370. at sun.misc.Unsafe.park(Native Method)
  371. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  372. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  373. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  374. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  375. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  376.  
  377. "IPC Server handler 30 on 60020" daemon prio=10 tid=0x00007f91a81e5800 nid=0x20df waiting on condition [0x00007f914a4e3000]
  378. java.lang.Thread.State: WAITING (parking)
  379. at sun.misc.Unsafe.park(Native Method)
  380. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  381. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  382. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  383. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  384. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  385.  
  386. "IPC Server handler 29 on 60020" daemon prio=10 tid=0x00007f91a81e3800 nid=0x20de waiting on condition [0x00007f914a5e4000]
  387. java.lang.Thread.State: WAITING (parking)
  388. at sun.misc.Unsafe.park(Native Method)
  389. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  390. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  391. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  392. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  393. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  394.  
  395. "IPC Server handler 28 on 60020" daemon prio=10 tid=0x00007f91a81e1800 nid=0x20dd runnable [0x00007f914a6e5000]
  396. java.lang.Thread.State: RUNNABLE
  397. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3113)
  398. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3108)
  399. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:74)
  400. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:62)
  401. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.getNext(MemStore.java:561)
  402. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:605)
  403. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:240)
  404. at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:382)
  405. - locked <0x000000050d1542b0> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  406. at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:288)
  407. - locked <0x000000050d1542b0> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  408. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:114)
  409. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.nextInternal(HRegion.java:2284)
  410. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2241)
  411. - locked <0x000000050d1541b0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  412. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2257)
  413. - locked <0x000000050d1541b0> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  414. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2943)
  415. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2847)
  416. at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1746)
  417. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  418. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  419. at java.lang.reflect.Method.invoke(Method.java:597)
  420. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560)
  421. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027)
  422.  
  423. "IPC Server handler 27 on 60020" daemon prio=10 tid=0x00007f91a81df800 nid=0x20dc waiting on condition [0x00007f914a7e6000]
  424. java.lang.Thread.State: WAITING (parking)
  425. at sun.misc.Unsafe.park(Native Method)
  426. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  427. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  428. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  429. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  430. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  431.  
  432. "IPC Server handler 26 on 60020" daemon prio=10 tid=0x00007f91a81dd800 nid=0x20db waiting on condition [0x00007f914a8e7000]
  433. java.lang.Thread.State: WAITING (parking)
  434. at sun.misc.Unsafe.park(Native Method)
  435. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  436. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  437. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  438. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  439. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  440.  
  441. "IPC Server handler 25 on 60020" daemon prio=10 tid=0x00007f91a81db800 nid=0x20da waiting on condition [0x00007f914a9e8000]
  442. java.lang.Thread.State: WAITING (parking)
  443. at sun.misc.Unsafe.park(Native Method)
  444. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  445. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  446. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  447. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  448. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  449.  
  450. "IPC Server handler 24 on 60020" daemon prio=10 tid=0x00007f91a81d9800 nid=0x20d9 waiting on condition [0x00007f914aae9000]
  451. java.lang.Thread.State: WAITING (parking)
  452. at sun.misc.Unsafe.park(Native Method)
  453. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  454. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  455. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  456. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  457. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  458.  
  459. "IPC Server handler 23 on 60020" daemon prio=10 tid=0x00007f91a81d7800 nid=0x20d8 waiting on condition [0x00007f914abea000]
  460. java.lang.Thread.State: WAITING (parking)
  461. at sun.misc.Unsafe.park(Native Method)
  462. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  463. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  464. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  465. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  466. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  467.  
  468. "IPC Server handler 22 on 60020" daemon prio=10 tid=0x00007f91a81d5800 nid=0x20d7 waiting on condition [0x00007f914aceb000]
  469. java.lang.Thread.State: WAITING (parking)
  470. at sun.misc.Unsafe.park(Native Method)
  471. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  472. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  473. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  474. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  475. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  476.  
  477. "IPC Server handler 21 on 60020" daemon prio=10 tid=0x00007f91a81d3800 nid=0x20d6 waiting on condition [0x00007f914adec000]
  478. java.lang.Thread.State: WAITING (parking)
  479. at sun.misc.Unsafe.park(Native Method)
  480. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  481. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  482. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  483. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  484. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  485.  
  486. "IPC Server handler 20 on 60020" daemon prio=10 tid=0x00007f91a81d1800 nid=0x20d5 waiting on condition [0x00007f914aeed000]
  487. java.lang.Thread.State: WAITING (parking)
  488. at sun.misc.Unsafe.park(Native Method)
  489. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  490. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  491. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  492. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  493. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  494.  
  495. "IPC Server handler 19 on 60020" daemon prio=10 tid=0x00007f91a81cf800 nid=0x20d4 waiting on condition [0x00007f914afee000]
  496. java.lang.Thread.State: WAITING (parking)
  497. at sun.misc.Unsafe.park(Native Method)
  498. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  499. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  500. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  501. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  502. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  503.  
  504. "IPC Server handler 18 on 60020" daemon prio=10 tid=0x00007f91a81cd800 nid=0x20d3 waiting on condition [0x00007f914b0ef000]
  505. java.lang.Thread.State: WAITING (parking)
  506. at sun.misc.Unsafe.park(Native Method)
  507. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  508. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  509. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  510. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  511. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  512.  
  513. "IPC Server handler 17 on 60020" daemon prio=10 tid=0x00007f91a81cb800 nid=0x20d2 runnable [0x00007f914b1f0000]
  514. java.lang.Thread.State: RUNNABLE
  515. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3113)
  516. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3108)
  517. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:74)
  518. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:62)
  519. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.getNext(MemStore.java:561)
  520. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:605)
  521. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:240)
  522. at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:382)
  523. - locked <0x000000050deb81e0> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  524. at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:288)
  525. - locked <0x000000050deb81e0> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  526. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:114)
  527. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.nextInternal(HRegion.java:2284)
  528. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2241)
  529. - locked <0x000000050deb8248> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  530. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2257)
  531. - locked <0x000000050deb8248> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  532. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2943)
  533. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2847)
  534. at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1746)
  535. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  536. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  537. at java.lang.reflect.Method.invoke(Method.java:597)
  538. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560)
  539. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027)
  540.  
  541. "IPC Server handler 16 on 60020" daemon prio=10 tid=0x00007f91a81c9800 nid=0x20d1 waiting on condition [0x00007f914b2f1000]
  542. java.lang.Thread.State: WAITING (parking)
  543. at sun.misc.Unsafe.park(Native Method)
  544. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  545. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  546. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  547. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  548. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  549.  
  550. "IPC Server handler 15 on 60020" daemon prio=10 tid=0x00007f91a81c7800 nid=0x20d0 waiting on condition [0x00007f914b3f2000]
  551. java.lang.Thread.State: WAITING (parking)
  552. at sun.misc.Unsafe.park(Native Method)
  553. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  554. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  555. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  556. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  557. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  558.  
  559. "IPC Server handler 14 on 60020" daemon prio=10 tid=0x00007f91a81c5800 nid=0x20cf waiting on condition [0x00007f914b4f3000]
  560. java.lang.Thread.State: WAITING (parking)
  561. at sun.misc.Unsafe.park(Native Method)
  562. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  563. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  564. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  565. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  566. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  567.  
  568. "IPC Server handler 13 on 60020" daemon prio=10 tid=0x00007f91a81c4000 nid=0x20ce waiting on condition [0x00007f914b5f4000]
  569. java.lang.Thread.State: WAITING (parking)
  570. at sun.misc.Unsafe.park(Native Method)
  571. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  572. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  573. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  574. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  575. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  576.  
  577. "IPC Server handler 12 on 60020" daemon prio=10 tid=0x00007f91a81c2000 nid=0x20cd waiting on condition [0x00007f914b6f5000]
  578. java.lang.Thread.State: WAITING (parking)
  579. at sun.misc.Unsafe.park(Native Method)
  580. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  581. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  582. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  583. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  584. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  585.  
  586. "IPC Server handler 11 on 60020" daemon prio=10 tid=0x00007f91a81c0000 nid=0x20cc waiting on condition [0x00007f914b7f6000]
  587. java.lang.Thread.State: WAITING (parking)
  588. at sun.misc.Unsafe.park(Native Method)
  589. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  590. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  591. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  592. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  593. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  594.  
  595. "IPC Server handler 10 on 60020" daemon prio=10 tid=0x00007f91a81be000 nid=0x20cb waiting on condition [0x00007f914b8f7000]
  596. java.lang.Thread.State: WAITING (parking)
  597. at sun.misc.Unsafe.park(Native Method)
  598. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  599. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  600. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  601. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  602. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  603.  
  604. "IPC Server handler 9 on 60020" daemon prio=10 tid=0x00007f91a81bc000 nid=0x20ca waiting on condition [0x00007f914b9f8000]
  605. java.lang.Thread.State: WAITING (parking)
  606. at sun.misc.Unsafe.park(Native Method)
  607. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  608. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  609. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  610. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  611. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  612.  
  613. "IPC Server handler 8 on 60020" daemon prio=10 tid=0x00007f91a81ba000 nid=0x20c9 waiting on condition [0x00007f914baf9000]
  614. java.lang.Thread.State: WAITING (parking)
  615. at sun.misc.Unsafe.park(Native Method)
  616. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  617. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  618. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  619. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  620. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  621.  
  622. "IPC Server handler 7 on 60020" daemon prio=10 tid=0x00007f91a81b8000 nid=0x20c8 runnable [0x00007f914bbfa000]
  623. java.lang.Thread.State: RUNNABLE
  624. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3113)
  625. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3108)
  626. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:74)
  627. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:62)
  628. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.getNext(MemStore.java:561)
  629. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:605)
  630. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:240)
  631. at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:382)
  632. - locked <0x000000050dec1b60> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  633. at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:288)
  634. - locked <0x000000050dec1b60> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  635. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:114)
  636. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.nextInternal(HRegion.java:2284)
  637. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2241)
  638. - locked <0x000000050dec1bc8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  639. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2257)
  640. - locked <0x000000050dec1bc8> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  641. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2943)
  642. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2847)
  643. at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1746)
  644. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  645. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  646. at java.lang.reflect.Method.invoke(Method.java:597)
  647. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560)
  648. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027)
  649.  
  650. "IPC Server handler 6 on 60020" daemon prio=10 tid=0x00007f91a81b6000 nid=0x20c7 waiting on condition [0x00007f914bcfb000]
  651. java.lang.Thread.State: WAITING (parking)
  652. at sun.misc.Unsafe.park(Native Method)
  653. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  654. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  655. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  656. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  657. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  658.  
  659. "IPC Server handler 5 on 60020" daemon prio=10 tid=0x00007f91a81b4000 nid=0x20c6 runnable [0x00007f914bdfc000]
  660. java.lang.Thread.State: RUNNABLE
  661. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3113)
  662. at java.util.concurrent.ConcurrentSkipListMap$SubMap$SubMapEntryIterator.next(ConcurrentSkipListMap.java:3108)
  663. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:74)
  664. at org.apache.hadoop.hbase.regionserver.KeyValueSkipListSet$MapEntryIterator.next(KeyValueSkipListSet.java:62)
  665. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.getNext(MemStore.java:561)
  666. at org.apache.hadoop.hbase.regionserver.MemStore$MemStoreScanner.reseek(MemStore.java:605)
  667. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.reseek(KeyValueHeap.java:240)
  668. at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:382)
  669. - locked <0x000000050debcea8> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  670. at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:288)
  671. - locked <0x000000050debcea8> (a org.apache.hadoop.hbase.regionserver.StoreScanner)
  672. at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:114)
  673. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.nextInternal(HRegion.java:2284)
  674. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2241)
  675. - locked <0x000000050debcf10> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  676. at org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner.next(HRegion.java:2257)
  677. - locked <0x000000050debcf10> (a org.apache.hadoop.hbase.regionserver.HRegion$RegionScanner)
  678. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2943)
  679. at org.apache.hadoop.hbase.regionserver.HRegion.get(HRegion.java:2847)
  680. at org.apache.hadoop.hbase.regionserver.HRegionServer.get(HRegionServer.java:1746)
  681. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  682. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  683. at java.lang.reflect.Method.invoke(Method.java:597)
  684. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:560)
  685. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1027)
  686.  
  687. "IPC Server handler 4 on 60020" daemon prio=10 tid=0x00007f91a81b2000 nid=0x20c5 waiting on condition [0x00007f914befd000]
  688. java.lang.Thread.State: WAITING (parking)
  689. at sun.misc.Unsafe.park(Native Method)
  690. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  691. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  692. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  693. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  694. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  695.  
  696. "IPC Server handler 3 on 60020" daemon prio=10 tid=0x00007f91a81b0800 nid=0x20c4 waiting on condition [0x00007f914bffe000]
  697. java.lang.Thread.State: WAITING (parking)
  698. at sun.misc.Unsafe.park(Native Method)
  699. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  700. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  701. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  702. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  703. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  704.  
  705. "IPC Server handler 2 on 60020" daemon prio=10 tid=0x00007f91a81ae800 nid=0x20c3 waiting on condition [0x00007f91f0132000]
  706. java.lang.Thread.State: WAITING (parking)
  707. at sun.misc.Unsafe.park(Native Method)
  708. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  709. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  710. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  711. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  712. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  713.  
  714. "IPC Server handler 1 on 60020" daemon prio=10 tid=0x00007f91a81ad800 nid=0x20c2 waiting on condition [0x00007f91f0233000]
  715. java.lang.Thread.State: WAITING (parking)
  716. at sun.misc.Unsafe.park(Native Method)
  717. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  718. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  719. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  720. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  721. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  722.  
  723. "IPC Server handler 0 on 60020" daemon prio=10 tid=0x00007f91a818c800 nid=0x20c1 waiting on condition [0x00007f91f0334000]
  724. java.lang.Thread.State: WAITING (parking)
  725. at sun.misc.Unsafe.park(Native Method)
  726. - parking to wait for <0x0000000518882148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  727. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  728. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  729. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  730. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1013)
  731.  
  732. "IPC Server listener on 60020" daemon prio=10 tid=0x00007f91a80bf000 nid=0x20c0 runnable [0x00007f91f0435000]
  733. java.lang.Thread.State: RUNNABLE
  734. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  735. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  736. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  737. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  738. - locked <0x0000000518888990> (a sun.nio.ch.Util$2)
  739. - locked <0x00000005188889a8> (a java.util.Collections$UnmodifiableSet)
  740. - locked <0x0000000518887c90> (a sun.nio.ch.EPollSelectorImpl)
  741. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  742. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  743. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:402)
  744.  
  745. "IPC Server Responder" daemon prio=10 tid=0x00007f91a80b6800 nid=0x20bf runnable [0x00007f91f0536000]
  746. java.lang.Thread.State: RUNNABLE
  747. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  748. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  749. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  750. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  751. - locked <0x0000000518883b78> (a sun.nio.ch.Util$2)
  752. - locked <0x0000000518883b90> (a java.util.Collections$UnmodifiableSet)
  753. - locked <0x0000000518889910> (a sun.nio.ch.EPollSelectorImpl)
  754. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  755. at org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:576)
  756.  
  757. "Timer-0" daemon prio=10 tid=0x00007f91a809c800 nid=0x20be in Object.wait() [0x00007f91f0637000]
  758. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  759. at java.lang.Object.wait(Native Method)
  760. at java.util.TimerThread.mainLoop(Timer.java:509)
  761. - locked <0x000000051888a438> (a java.util.TaskQueue)
  762. at java.util.TimerThread.run(Timer.java:462)
  763.  
  764. "389362249@qtp-877230050-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030" prio=10 tid=0x00007f91a8086800 nid=0x20bd runnable [0x00007f91f0738000]
  765. java.lang.Thread.State: RUNNABLE
  766. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  767. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  768. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  769. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  770. - locked <0x000000051889b520> (a sun.nio.ch.Util$2)
  771. - locked <0x000000051889b538> (a java.util.Collections$UnmodifiableSet)
  772. - locked <0x00000005188bdf28> (a sun.nio.ch.EPollSelectorImpl)
  773. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  774. at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:484)
  775. at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:191)
  776. at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
  777. at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
  778. at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
  779.  
  780. "regionserver60020.leaseChecker" prio=10 tid=0x00007f91a807b800 nid=0x20bb waiting on condition [0x00007f91f093a000]
  781. java.lang.Thread.State: TIMED_WAITING (parking)
  782. at sun.misc.Unsafe.park(Native Method)
  783. - parking to wait for <0x00000005188b6390> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  784. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  785. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  786. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:209)
  787. at org.apache.hadoop.hbase.Leases.run(Leases.java:81)
  788.  
  789. "regionserver60020.majorCompactionChecker" daemon prio=10 tid=0x00007f91a8079800 nid=0x20ba in Object.wait() [0x00007f91f0a3b000]
  790. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  791. at java.lang.Object.wait(Native Method)
  792. at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
  793. - locked <0x00000005188831c8> (a java.lang.Object)
  794. at org.apache.hadoop.hbase.Chore.run(Chore.java:76)
  795.  
  796. "regionserver60020.worker" daemon prio=10 tid=0x00007f91a8077800 nid=0x20b9 waiting on condition [0x00007f91f0b3c000]
  797. java.lang.Thread.State: TIMED_WAITING (parking)
  798. at sun.misc.Unsafe.park(Native Method)
  799. - parking to wait for <0x0000000518889210> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  800. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  801. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  802. at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:424)
  803. at org.apache.hadoop.hbase.regionserver.HRegionServer$Worker.run(HRegionServer.java:1428)
  804. at java.lang.Thread.run(Thread.java:662)
  805.  
  806. "regionserver60020.compactor" daemon prio=10 tid=0x00007f91a8075800 nid=0x20b8 waiting on condition [0x00007f91f0c3d000]
  807. java.lang.Thread.State: TIMED_WAITING (parking)
  808. at sun.misc.Unsafe.park(Native Method)
  809. - parking to wait for <0x00000005188889c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  810. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  811. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  812. at java.util.concurrent.PriorityBlockingQueue.poll(PriorityBlockingQueue.java:245)
  813. at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue.poll(PriorityCompactionQueue.java:244)
  814. at org.apache.hadoop.hbase.regionserver.CompactSplitThread.run(CompactSplitThread.java:99)
  815.  
  816. "regionserver60020.cacheFlusher" daemon prio=10 tid=0x00007f91a8074000 nid=0x20b7 waiting on condition [0x00007f91f0d3e000]
  817. java.lang.Thread.State: TIMED_WAITING (parking)
  818. at sun.misc.Unsafe.park(Native Method)
  819. - parking to wait for <0x0000000518893110> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  820. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  821. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  822. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
  823. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:39)
  824. at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:142)
  825.  
  826. "regionserver60020.logRoller" daemon prio=10 tid=0x00007f91a8070000 nid=0x20b6 in Object.wait() [0x00007f91f0e3f000]
  827. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  828. at java.lang.Object.wait(Native Method)
  829. at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:67)
  830. - locked <0x0000000518883ba8> (a java.util.concurrent.atomic.AtomicBoolean)
  831.  
  832. "Timer thread for monitoring jvm" daemon prio=10 tid=0x00007f91a806f000 nid=0x20b5 in Object.wait() [0x00007f91f0f40000]
  833. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  834. at java.lang.Object.wait(Native Method)
  835. at java.util.TimerThread.mainLoop(Timer.java:509)
  836. - locked <0x00000005188c0f78> (a java.util.TaskQueue)
  837. at java.util.TimerThread.run(Timer.java:462)
  838.  
  839. "Timer thread for monitoring hbase" daemon prio=10 tid=0x00007f91a806a800 nid=0x20b4 in Object.wait() [0x00007f91f1041000]
  840. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  841. at java.lang.Object.wait(Native Method)
  842. at java.util.TimerThread.mainLoop(Timer.java:509)
  843. - locked <0x000000051888a4e0> (a java.util.TaskQueue)
  844. at java.util.TimerThread.run(Timer.java:462)
  845.  
  846. "regionserver60020.logSyncer" daemon prio=10 tid=0x00007f91a8069800 nid=0x20b3 waiting on condition [0x00007f91f1142000]
  847. java.lang.Thread.State: TIMED_WAITING (sleeping)
  848. at java.lang.Thread.sleep(Native Method)
  849. at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:882)
  850.  
  851. "LeaseChecker" daemon prio=10 tid=0x00007f91a805a800 nid=0x20b2 waiting on condition [0x00007f91f1243000]
  852. java.lang.Thread.State: TIMED_WAITING (sleeping)
  853. at java.lang.Thread.sleep(Native Method)
  854. at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1071)
  855. at java.lang.Thread.run(Thread.java:662)
  856.  
  857. "sendParams-0" daemon prio=10 tid=0x00007f91a8035000 nid=0x20af waiting on condition [0x00007f91f1546000]
  858. java.lang.Thread.State: TIMED_WAITING (parking)
  859. at sun.misc.Unsafe.park(Native Method)
  860. - parking to wait for <0x000000051888a348> (a java.util.concurrent.SynchronousQueue$TransferStack)
  861. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  862. at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
  863. at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
  864. at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
  865. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
  866. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  867. at java.lang.Thread.run(Thread.java:662)
  868.  
  869. "DestroyJavaVM" prio=10 tid=0x00007f924c008800 nid=0x2080 waiting on condition [0x0000000000000000]
  870. java.lang.Thread.State: RUNNABLE
  871.  
  872. "regionserver60020" prio=10 tid=0x00007f924c5e6800 nid=0x20a3 waiting on condition [0x00007f91f1849000]
  873. java.lang.Thread.State: TIMED_WAITING (parking)
  874. at sun.misc.Unsafe.park(Native Method)
  875. - parking to wait for <0x000000051890fa88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  876. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  877. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  878. at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:424)
  879. at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:647)
  880. at java.lang.Thread.run(Thread.java:662)
  881.  
  882. "main-EventThread" daemon prio=10 tid=0x00007f924c5a8000 nid=0x20a2 waiting on condition [0x00007f91f194a000]
  883. java.lang.Thread.State: WAITING (parking)
  884. at sun.misc.Unsafe.park(Native Method)
  885. - parking to wait for <0x0000000518893158> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  886. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  887. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  888. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  889. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:477)
  890.  
  891. "main-SendThread(mtag2:2181)" daemon prio=10 tid=0x00007f924c5e3000 nid=0x20a1 runnable [0x00007f91f1a4b000]
  892. java.lang.Thread.State: RUNNABLE
  893. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  894. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  895. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  896. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  897. - locked <0x0000000518883bc0> (a sun.nio.ch.Util$2)
  898. - locked <0x0000000518883bd8> (a java.util.Collections$UnmodifiableSet)
  899. - locked <0x0000000518889958> (a sun.nio.ch.EPollSelectorImpl)
  900. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  901. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1066)
  902.  
  903. "Timer thread for monitoring rpc" daemon prio=10 tid=0x00007f924c599000 nid=0x20a0 in Object.wait() [0x00007f91f1f6b000]
  904. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  905. at java.lang.Object.wait(Native Method)
  906. at java.util.TimerThread.mainLoop(Timer.java:509)
  907. - locked <0x00000005188931a0> (a java.util.TaskQueue)
  908. at java.util.TimerThread.run(Timer.java:462)
  909.  
  910. "pool-1-thread-10" prio=10 tid=0x00007f924c596800 nid=0x209f runnable [0x00007f91f206c000]
  911. java.lang.Thread.State: RUNNABLE
  912. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  913. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  914. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  915. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  916. - locked <0x00000005188931d0> (a sun.nio.ch.Util$2)
  917. - locked <0x00000005188931e8> (a java.util.Collections$UnmodifiableSet)
  918. - locked <0x000000051889d9d0> (a sun.nio.ch.EPollSelectorImpl)
  919. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  920. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  921. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  922. - locked <0x0000000518893200> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  923. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  924. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  925. at java.lang.Thread.run(Thread.java:662)
  926.  
  927. "pool-1-thread-9" prio=10 tid=0x00007f924c57b800 nid=0x209e runnable [0x00007f91f216d000]
  928. java.lang.Thread.State: RUNNABLE
  929. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  930. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  931. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  932. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  933. - locked <0x0000000518883bf0> (a sun.nio.ch.Util$2)
  934. - locked <0x0000000518883c08> (a java.util.Collections$UnmodifiableSet)
  935. - locked <0x00000005188899a0> (a sun.nio.ch.EPollSelectorImpl)
  936. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  937. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  938. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  939. - locked <0x0000000518883c20> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  940. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  941. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  942. at java.lang.Thread.run(Thread.java:662)
  943.  
  944. "pool-1-thread-8" prio=10 tid=0x00007f924c561000 nid=0x209d runnable [0x00007f92141bf000]
  945. java.lang.Thread.State: RUNNABLE
  946. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  947. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  948. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  949. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  950. - locked <0x0000000518893230> (a sun.nio.ch.Util$2)
  951. - locked <0x0000000518893248> (a java.util.Collections$UnmodifiableSet)
  952. - locked <0x000000051889da18> (a sun.nio.ch.EPollSelectorImpl)
  953. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  954. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  955. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  956. - locked <0x0000000518893260> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  957. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  958. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  959. at java.lang.Thread.run(Thread.java:662)
  960.  
  961. "pool-1-thread-7" prio=10 tid=0x00007f924c546000 nid=0x209c runnable [0x00007f92142c0000]
  962. java.lang.Thread.State: RUNNABLE
  963. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  964. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  965. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  966. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  967. - locked <0x0000000518883c50> (a sun.nio.ch.Util$2)
  968. - locked <0x0000000518883c68> (a java.util.Collections$UnmodifiableSet)
  969. - locked <0x00000005188899e8> (a sun.nio.ch.EPollSelectorImpl)
  970. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  971. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  972. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  973. - locked <0x0000000518883c80> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  974. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  975. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  976. at java.lang.Thread.run(Thread.java:662)
  977.  
  978. "pool-1-thread-6" prio=10 tid=0x00007f924c52b000 nid=0x209b runnable [0x00007f92143c1000]
  979. java.lang.Thread.State: RUNNABLE
  980. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  981. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  982. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  983. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  984. - locked <0x000000051889e0d0> (a sun.nio.ch.Util$2)
  985. - locked <0x000000051889e0e8> (a java.util.Collections$UnmodifiableSet)
  986. - locked <0x000000051889da60> (a sun.nio.ch.EPollSelectorImpl)
  987. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  988. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  989. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  990. - locked <0x000000051889e100> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  991. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  992. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  993. at java.lang.Thread.run(Thread.java:662)
  994.  
  995. "pool-1-thread-5" prio=10 tid=0x00007f924c510000 nid=0x209a runnable [0x00007f92144c2000]
  996. java.lang.Thread.State: RUNNABLE
  997. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  998. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  999. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  1000. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  1001. - locked <0x0000000518883cb0> (a sun.nio.ch.Util$2)
  1002. - locked <0x0000000518883cc8> (a java.util.Collections$UnmodifiableSet)
  1003. - locked <0x0000000518889a30> (a sun.nio.ch.EPollSelectorImpl)
  1004. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  1005. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  1006. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  1007. - locked <0x000000051889e250> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  1008. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  1009. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  1010. at java.lang.Thread.run(Thread.java:662)
  1011.  
  1012. "pool-1-thread-4" prio=10 tid=0x00007f924c4f5800 nid=0x2099 runnable [0x00007f92145c3000]
  1013. java.lang.Thread.State: RUNNABLE
  1014. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1015. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  1016. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  1017. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  1018. - locked <0x000000051889e130> (a sun.nio.ch.Util$2)
  1019. - locked <0x000000051889e148> (a java.util.Collections$UnmodifiableSet)
  1020. - locked <0x000000051889daa8> (a sun.nio.ch.EPollSelectorImpl)
  1021. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  1022. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  1023. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  1024. - locked <0x000000051889e160> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  1025. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  1026. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  1027. at java.lang.Thread.run(Thread.java:662)
  1028.  
  1029. "pool-1-thread-3" prio=10 tid=0x00007f924c4db000 nid=0x2098 runnable [0x00007f92146c4000]
  1030. java.lang.Thread.State: RUNNABLE
  1031. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1032. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  1033. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  1034. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  1035. - locked <0x000000051889e280> (a sun.nio.ch.Util$2)
  1036. - locked <0x000000051889e298> (a java.util.Collections$UnmodifiableSet)
  1037. - locked <0x0000000518889a78> (a sun.nio.ch.EPollSelectorImpl)
  1038. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  1039. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  1040. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  1041. - locked <0x000000051889e2b0> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  1042. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  1043. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  1044. at java.lang.Thread.run(Thread.java:662)
  1045.  
  1046. "pool-1-thread-2" prio=10 tid=0x00007f924c461000 nid=0x2097 runnable [0x00007f92147c5000]
  1047. java.lang.Thread.State: RUNNABLE
  1048. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1049. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  1050. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  1051. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  1052. - locked <0x000000051889e190> (a sun.nio.ch.Util$2)
  1053. - locked <0x000000051889e1a8> (a java.util.Collections$UnmodifiableSet)
  1054. - locked <0x000000051889daf0> (a sun.nio.ch.EPollSelectorImpl)
  1055. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  1056. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  1057. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  1058. - locked <0x000000051889e1c0> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  1059. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  1060. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  1061. at java.lang.Thread.run(Thread.java:662)
  1062.  
  1063. "pool-1-thread-1" prio=10 tid=0x00007f924c266800 nid=0x2096 runnable [0x00007f92148c6000]
  1064. java.lang.Thread.State: RUNNABLE
  1065. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1066. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  1067. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  1068. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  1069. - locked <0x000000051889e1f0> (a sun.nio.ch.Util$2)
  1070. - locked <0x000000051889e208> (a java.util.Collections$UnmodifiableSet)
  1071. - locked <0x000000051889db38> (a sun.nio.ch.EPollSelectorImpl)
  1072. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  1073. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:84)
  1074. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:293)
  1075. - locked <0x000000051889e238> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  1076. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  1077. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  1078. at java.lang.Thread.run(Thread.java:662)
  1079.  
  1080. "Low Memory Detector" daemon prio=10 tid=0x00007f924c222800 nid=0x2093 runnable [0x0000000000000000]
  1081. java.lang.Thread.State: RUNNABLE
  1082.  
  1083. "CompilerThread1" daemon prio=10 tid=0x00007f924c220000 nid=0x2092 waiting on condition [0x0000000000000000]
  1084. java.lang.Thread.State: RUNNABLE
  1085.  
  1086. "CompilerThread0" daemon prio=10 tid=0x00007f924c21e000 nid=0x2091 waiting on condition [0x0000000000000000]
  1087. java.lang.Thread.State: RUNNABLE
  1088. at java.lang.Object.wait(Object.java:485)
  1089. at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
  1090. - locked <0x0000000518888a38> (a java.lang.ref.Reference$Lock)
  1091.  
  1092. "VM Thread" prio=10 tid=0x00007f924c1f4800 nid=0x208c runnable
  1093.  
  1094. "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x00007f924c017000 nid=0x2081 runnable
  1095.  
  1096. "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x00007f924c019000 nid=0x2082 runnable
  1097.  
  1098. "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x00007f924c01a800 nid=0x2083 runnable
  1099.  
  1100. "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x00007f924c01c800 nid=0x2084 runnable
  1101.  
  1102. "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x00007f924c01e000 nid=0x2085 runnable
  1103.  
  1104. "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x00007f924c020000 nid=0x2086 runnable
  1105.  
  1106. "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x00007f924c022000 nid=0x2087 runnable
  1107.  
  1108. "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x00007f924c023800 nid=0x2088 runnable
  1109.  
  1110. "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x00007f924c117800 nid=0x208b runnable
  1111. "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x00007f924c113800 nid=0x2089 runnable
  1112.  
  1113. "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x00007f924c115800 nid=0x208a runnable
  1114.  
  1115. "VM Periodic Task Thread" prio=10 tid=0x00007f924c235800 nid=0x2094 waiting on condition
  1116.  
  1117. JNI global references: 1720
  1118.  
  1119. Heap
  1120. par new generation total 19136K, used 15314K [0x000000050ce00000, 0x000000050e2c0000, 0x0000000517460000)
  1121. eden space 17024K, 87% used [0x000000050ce00000, 0x000000050dc7fde8, 0x000000050dea0000)
  1122. from space 2112K, 22% used [0x000000050dea0000, 0x000000050df14a28, 0x000000050e0b0000)
  1123. to space 2112K, 0% used [0x000000050e0b0000, 0x000000050e0b0000, 0x000000050e2c0000)
  1124. concurrent mark-sweep generation total 11894296K, used 6994552K [0x0000000517460000, 0x00000007ed3e6000, 0x00000007fae00000)
  1125. concurrent-mark-sweep perm gen total 31240K, used 18741K [0x00000007fae00000, 0x00000007fcc82000, 0x0000000800000000)