API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Feb 29th, 2012
80
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 195.02 KB | None | 0 0
raw download clone embed print report
  1. 2012-02-29 13:12:44
  2. Full thread dump Java HotSpot(TM) 64-Bit Server VM (19.0-b09 mixed mode):
  3.  
  4. "Thread-181690" daemon prio=10 tid=0x00007f983434a800 nid=0x6f7b in Object.wait() [0x00007f982cc8b000]
  5. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  6. at java.lang.Object.wait(Native Method)
  7. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2466)
  8. - locked <0x000000041c3c0080> (a java.util.LinkedList)
  9.  
  10. "Attach Listener" daemon prio=10 tid=0x0000000042bab000 nid=0x6c22 waiting on condition [0x0000000000000000]
  11. java.lang.Thread.State: RUNNABLE
  12.  
  13. "IPC Client (47) connection to /192.168.201.13:50020 from blk_-8859664738058583740_69352641" daemon prio=10 tid=0x00000000423cd000 nid=0x435c runnable [0x00007f982c564000]
  14. java.lang.Thread.State: RUNNABLE
  15. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  16. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  17. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  18. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  19. - locked <0x00000005bc073bb8> (a sun.nio.ch.Util$2)
  20. - locked <0x00000005bc073ba0> (a java.util.Collections$UnmodifiableSet)
  21. - locked <0x00000005bc4fe9c8> (a sun.nio.ch.EPollSelectorImpl)
  22. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  23. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:332)
  24. at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  25. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:155)
  26. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:128)
  27. at java.io.FilterInputStream.read(FilterInputStream.java:116)
  28. at org.apache.hadoop.ipc.Client$Connection$PingInputStream.read(Client.java:368)
  29. at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
  30. at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
  31. - locked <0x00000005c2b52868> (a java.io.BufferedInputStream)
  32. at java.io.DataInputStream.readInt(DataInputStream.java:370)
  33. at org.apache.hadoop.ipc.Client$Connection.receiveResponse(Client.java:781)
  34. at org.apache.hadoop.ipc.Client$Connection.run(Client.java:689)
  35.  
  36. "DataStreamer for file /offline-hbase/.logs/prod00013.soniu.com,60020,1327993969023/prod00013.soniu.com%3A60020.1330488335312 block blk_-8859664738058583740_69352641" daemon prio=10 tid=0x00007f98343ec000 nid=0x6c4e in Object.wait() [0x00007f982d695000]
  37. java.lang.Thread.State: WAITING (on object monitor)
  38. at java.lang.Object.wait(Native Method)
  39. at java.lang.Object.wait(Object.java:485)
  40. at org.apache.hadoop.ipc.Client.call(Client.java:1040)
  41. - locked <0x00000005c2b1f3d0> (a org.apache.hadoop.ipc.Client$Call)
  42. at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:226)
  43. at $Proxy9.recoverBlock(Unknown Source)
  44. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.processDatanodeError(DFSClient.java:2782)
  45. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.access$1600(DFSClient.java:2288)
  46. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream$DataStreamer.run(DFSClient.java:2460)
  47. - locked <0x000000059943a5d8> (a java.util.LinkedList)
  48.  
  49. "IPC Client (47) connection to prod00004.soniu.com/192.168.201.4:60000 from an unknown user" daemon prio=10 tid=0x00007f9828014000 nid=0x5cb0 in Object.wait() [0x00007f9839904000]
  50. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  51. at java.lang.Object.wait(Native Method)
  52. at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.waitForWork(HBaseClient.java:431)
  53. - locked <0x00000004292ac0d8> (a org.apache.hadoop.hbase.ipc.HBaseClient$Connection)
  54. at org.apache.hadoop.hbase.ipc.HBaseClient$Connection.run(HBaseClient.java:476)
  55.  
  56. "sendParams-4" daemon prio=10 tid=0x00000000423c4000 nid=0x66cb waiting on condition [0x00007f982d190000]
  57. java.lang.Thread.State: TIMED_WAITING (parking)
  58. at sun.misc.Unsafe.park(Native Method)
  59. - parking to wait for <0x000000041d97cdd8> (a java.util.concurrent.SynchronousQueue$TransferStack)
  60. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  61. at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:424)
  62. at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:323)
  63. at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:874)
  64. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:945)
  65. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  66. at java.lang.Thread.run(Thread.java:662)
  67.  
  68. "RS_CLOSE_REGION-prod00013.soniu.com,60020,1327993969023-2" daemon prio=10 tid=0x0000000044ac6800 nid=0x7915 waiting on condition [0x00007f982d291000]
  69. java.lang.Thread.State: WAITING (parking)
  70. at sun.misc.Unsafe.park(Native Method)
  71. - parking to wait for <0x000000041ea8bec8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  72. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  73. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  74. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  75. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  76. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  77. at java.lang.Thread.run(Thread.java:662)
  78.  
  79. "RS_CLOSE_REGION-prod00013.soniu.com,60020,1327993969023-1" daemon prio=10 tid=0x0000000042565800 nid=0x790f waiting on condition [0x00007f982ce8d000]
  80. java.lang.Thread.State: WAITING (parking)
  81. at sun.misc.Unsafe.park(Native Method)
  82. - parking to wait for <0x000000041ea8bec8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  83. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  84. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  85. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  86. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  87. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  88. at java.lang.Thread.run(Thread.java:662)
  89.  
  90. "RS_CLOSE_REGION-prod00013.soniu.com,60020,1327993969023-0" daemon prio=10 tid=0x0000000043601800 nid=0x790e waiting on condition [0x00007f982d08f000]
  91. java.lang.Thread.State: WAITING (parking)
  92. at sun.misc.Unsafe.park(Native Method)
  93. - parking to wait for <0x000000041ea8bec8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  94. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  95. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  96. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  97. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  98. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  99. at java.lang.Thread.run(Thread.java:662)
  100.  
  101. "RS_OPEN_REGION-prod00013.soniu.com,60020,1327993969023-2" daemon prio=10 tid=0x0000000042228800 nid=0x1e55 waiting on condition [0x00007f982d392000]
  102. java.lang.Thread.State: WAITING (parking)
  103. at sun.misc.Unsafe.park(Native Method)
  104. - parking to wait for <0x000000041eaa21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  105. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  106. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  107. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  108. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  109. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  110. at java.lang.Thread.run(Thread.java:662)
  111.  
  112. "RS_OPEN_REGION-prod00013.soniu.com,60020,1327993969023-1" daemon prio=10 tid=0x0000000041c94000 nid=0x1e4c waiting on condition [0x00007f982d493000]
  113. java.lang.Thread.State: WAITING (parking)
  114. at sun.misc.Unsafe.park(Native Method)
  115. - parking to wait for <0x000000041eaa21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  116. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  117. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  118. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  119. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  120. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  121. at java.lang.Thread.run(Thread.java:662)
  122.  
  123. "RS_OPEN_REGION-prod00013.soniu.com,60020,1327993969023-0" daemon prio=10 tid=0x00000000424cd800 nid=0x1e44 waiting on condition [0x00007f9839a05000]
  124. java.lang.Thread.State: WAITING (parking)
  125. at sun.misc.Unsafe.park(Native Method)
  126. - parking to wait for <0x000000041eaa21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  127. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  128. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  129. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  130. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  131. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  132. at java.lang.Thread.run(Thread.java:662)
  133.  
  134. "LRU Statistics #0" prio=10 tid=0x00007f983c551000 nid=0x1b84 waiting on condition [0x00007f982d796000]
  135. java.lang.Thread.State: TIMED_WAITING (parking)
  136. at sun.misc.Unsafe.park(Native Method)
  137. - parking to wait for <0x000000041ea720f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  138. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  139. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  140. at java.util.concurrent.DelayQueue.take(DelayQueue.java:164)
  141. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:609)
  142. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:602)
  143. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
  144. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
  145. at java.lang.Thread.run(Thread.java:662)
  146.  
  147. "LruBlockCache.EvictionThread" daemon prio=10 tid=0x00007f983c54f000 nid=0x1b83 in Object.wait() [0x00007f982d897000]
  148. java.lang.Thread.State: WAITING (on object monitor)
  149. at java.lang.Object.wait(Native Method)
  150. at java.lang.Object.wait(Object.java:485)
  151. at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:519)
  152. - locked <0x000000041e9f4420> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread)
  153.  
  154. "PRI IPC Server handler 9 on 60020" daemon prio=10 tid=0x00007f983c54d000 nid=0x1b80 waiting on condition [0x00007f982d998000]
  155. java.lang.Thread.State: WAITING (parking)
  156. at sun.misc.Unsafe.park(Native Method)
  157. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  158. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  159. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  160. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  161. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  162.  
  163. "PRI IPC Server handler 8 on 60020" daemon prio=10 tid=0x00007f983c54b000 nid=0x1b7f waiting on condition [0x00007f982da99000]
  164. java.lang.Thread.State: WAITING (parking)
  165. at sun.misc.Unsafe.park(Native Method)
  166. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  167. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  168. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  169. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  170. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  171.  
  172. "PRI IPC Server handler 7 on 60020" daemon prio=10 tid=0x00007f983c549000 nid=0x1b7e waiting on condition [0x00007f982db9a000]
  173. java.lang.Thread.State: WAITING (parking)
  174. at sun.misc.Unsafe.park(Native Method)
  175. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  176. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  177. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  178. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  179. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  180.  
  181. "PRI IPC Server handler 6 on 60020" daemon prio=10 tid=0x00007f983c547000 nid=0x1b7d waiting on condition [0x00007f982dc9b000]
  182. java.lang.Thread.State: WAITING (parking)
  183. at sun.misc.Unsafe.park(Native Method)
  184. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  185. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  186. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  187. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  188. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  189.  
  190. "PRI IPC Server handler 5 on 60020" daemon prio=10 tid=0x00007f983c545000 nid=0x1b7c waiting on condition [0x00007f982dd9c000]
  191. java.lang.Thread.State: WAITING (parking)
  192. at sun.misc.Unsafe.park(Native Method)
  193. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  194. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  195. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  196. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  197. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  198.  
  199. "PRI IPC Server handler 4 on 60020" daemon prio=10 tid=0x00007f983c543000 nid=0x1b7b waiting on condition [0x00007f982de9d000]
  200. java.lang.Thread.State: WAITING (parking)
  201. at sun.misc.Unsafe.park(Native Method)
  202. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  203. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  204. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  205. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  206. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  207.  
  208. "PRI IPC Server handler 3 on 60020" daemon prio=10 tid=0x00007f983c541000 nid=0x1b7a waiting on condition [0x00007f982df9e000]
  209. java.lang.Thread.State: WAITING (parking)
  210. at sun.misc.Unsafe.park(Native Method)
  211. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  212. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  213. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  214. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  215. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  216.  
  217. "PRI IPC Server handler 2 on 60020" daemon prio=10 tid=0x00007f983c53f000 nid=0x1b79 waiting on condition [0x00007f982e09f000]
  218. java.lang.Thread.State: WAITING (parking)
  219. at sun.misc.Unsafe.park(Native Method)
  220. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  221. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  222. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  223. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  224. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  225.  
  226. "PRI IPC Server handler 1 on 60020" daemon prio=10 tid=0x00007f983c53d000 nid=0x1b78 waiting on condition [0x00007f982e1a0000]
  227. java.lang.Thread.State: WAITING (parking)
  228. at sun.misc.Unsafe.park(Native Method)
  229. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  230. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  231. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  232. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  233. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  234.  
  235. "PRI IPC Server handler 0 on 60020" daemon prio=10 tid=0x00007f983c53b000 nid=0x1b77 waiting on condition [0x00007f982e2a1000]
  236. java.lang.Thread.State: WAITING (parking)
  237. at sun.misc.Unsafe.park(Native Method)
  238. - parking to wait for <0x000000041d9f5148> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  239. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  240. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  241. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  242. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1025)
  243.  
  244. "IPC Server handler 99 on 60020" daemon prio=10 tid=0x00007f983c539000 nid=0x1b76 in Object.wait() [0x00007f982e3a2000]
  245. java.lang.Thread.State: WAITING (on object monitor)
  246. at java.lang.Object.wait(Native Method)
  247. at java.lang.Object.wait(Object.java:485)
  248. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  249. - locked <0x000000059add8a28> (a java.util.LinkedList)
  250. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  251. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  252. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  253. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  254. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  255. at java.lang.reflect.Method.invoke(Method.java:597)
  256. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  257. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  258. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  259. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  260. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  261. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  262. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  263. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  264. at java.lang.reflect.Method.invoke(Method.java:597)
  265. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  266. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  267.  
  268. "IPC Server handler 98 on 60020" daemon prio=10 tid=0x00007f983c537000 nid=0x1b75 in Object.wait() [0x00007f982e4a3000]
  269. java.lang.Thread.State: WAITING (on object monitor)
  270. at java.lang.Object.wait(Native Method)
  271. at java.lang.Object.wait(Object.java:485)
  272. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  273. - locked <0x0000000420ec2fa0> (a java.util.TreeSet)
  274. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  275. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  276. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  277. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  278. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  279. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  280. at java.lang.reflect.Method.invoke(Method.java:597)
  281. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  282. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  283.  
  284. "IPC Server handler 97 on 60020" daemon prio=10 tid=0x00007f983c535000 nid=0x1b74 in Object.wait() [0x00007f982e5a4000]
  285. java.lang.Thread.State: WAITING (on object monitor)
  286. at java.lang.Object.wait(Native Method)
  287. at java.lang.Object.wait(Object.java:485)
  288. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  289. - locked <0x000000059add8a28> (a java.util.LinkedList)
  290. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  291. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  292. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  293. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  294. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  295. at java.lang.reflect.Method.invoke(Method.java:597)
  296. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  297. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  298. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  299. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  300. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  301. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  302. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  303. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  304. at java.lang.reflect.Method.invoke(Method.java:597)
  305. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  306. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  307.  
  308. "IPC Server handler 96 on 60020" daemon prio=10 tid=0x00007f983c533000 nid=0x1b73 in Object.wait() [0x00007f982e6a5000]
  309. java.lang.Thread.State: WAITING (on object monitor)
  310. at java.lang.Object.wait(Native Method)
  311. at java.lang.Object.wait(Object.java:485)
  312. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  313. - locked <0x000000059add8a28> (a java.util.LinkedList)
  314. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  315. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  316. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  317. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  318. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  319. at java.lang.reflect.Method.invoke(Method.java:597)
  320. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  321. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  322. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  323. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  324. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  325. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  326. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  327. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  328. at java.lang.reflect.Method.invoke(Method.java:597)
  329. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  330. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  331.  
  332. "IPC Server handler 95 on 60020" daemon prio=10 tid=0x00007f983c531000 nid=0x1b72 in Object.wait() [0x00007f982e7a6000]
  333. java.lang.Thread.State: WAITING (on object monitor)
  334. at java.lang.Object.wait(Native Method)
  335. at java.lang.Object.wait(Object.java:485)
  336. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  337. - locked <0x000000059add8a28> (a java.util.LinkedList)
  338. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  339. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  340. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  341. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  342. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  343. at java.lang.reflect.Method.invoke(Method.java:597)
  344. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  345. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  346. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  347. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  348. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  349. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  350. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  351. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  352. at java.lang.reflect.Method.invoke(Method.java:597)
  353. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  354. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  355.  
  356. "IPC Server handler 94 on 60020" daemon prio=10 tid=0x00007f983c52f000 nid=0x1b71 in Object.wait() [0x00007f982e8a7000]
  357. java.lang.Thread.State: WAITING (on object monitor)
  358. at java.lang.Object.wait(Native Method)
  359. at java.lang.Object.wait(Object.java:485)
  360. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  361. - locked <0x000000059add8a28> (a java.util.LinkedList)
  362. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  363. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  364. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  365. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  366. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  367. at java.lang.reflect.Method.invoke(Method.java:597)
  368. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  369. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  370. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  371. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  372. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  373. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  374. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  375. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  376. at java.lang.reflect.Method.invoke(Method.java:597)
  377. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  378. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  379.  
  380. "IPC Server handler 93 on 60020" daemon prio=10 tid=0x00007f983c52d000 nid=0x1b70 in Object.wait() [0x00007f982e9a8000]
  381. java.lang.Thread.State: WAITING (on object monitor)
  382. at java.lang.Object.wait(Native Method)
  383. at java.lang.Object.wait(Object.java:485)
  384. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  385. - locked <0x000000041f709b18> (a java.util.TreeSet)
  386. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  387. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  388. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  389. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  390. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  391. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  392. at java.lang.reflect.Method.invoke(Method.java:597)
  393. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  394. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  395.  
  396. "IPC Server handler 92 on 60020" daemon prio=10 tid=0x00007f983c52b000 nid=0x1b6f in Object.wait() [0x00007f982eaa9000]
  397. java.lang.Thread.State: WAITING (on object monitor)
  398. at java.lang.Object.wait(Native Method)
  399. at java.lang.Object.wait(Object.java:485)
  400. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  401. - locked <0x000000041f709b18> (a java.util.TreeSet)
  402. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  403. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  404. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  405. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  406. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  407. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  408. at java.lang.reflect.Method.invoke(Method.java:597)
  409. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  410. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  411.  
  412. "IPC Server handler 91 on 60020" daemon prio=10 tid=0x00007f983c529000 nid=0x1b6e in Object.wait() [0x00007f982ebaa000]
  413. java.lang.Thread.State: WAITING (on object monitor)
  414. at java.lang.Object.wait(Native Method)
  415. at java.lang.Object.wait(Object.java:485)
  416. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  417. - locked <0x000000059add8a28> (a java.util.LinkedList)
  418. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  419. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  420. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  421. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  422. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  423. at java.lang.reflect.Method.invoke(Method.java:597)
  424. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  425. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  426. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  427. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  428. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  429. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  430. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  431. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  432. at java.lang.reflect.Method.invoke(Method.java:597)
  433. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  434. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  435.  
  436. "IPC Server handler 90 on 60020" daemon prio=10 tid=0x00007f983c527000 nid=0x1b6d in Object.wait() [0x00007f982ecab000]
  437. java.lang.Thread.State: WAITING (on object monitor)
  438. at java.lang.Object.wait(Native Method)
  439. at java.lang.Object.wait(Object.java:485)
  440. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  441. - locked <0x00000004206e7390> (a java.util.TreeSet)
  442. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  443. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1172)
  444. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  445. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  446. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  447. at java.lang.reflect.Method.invoke(Method.java:597)
  448. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  449. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  450.  
  451. "IPC Server handler 89 on 60020" daemon prio=10 tid=0x00007f983c525000 nid=0x1b6c in Object.wait() [0x00007f982edac000]
  452. java.lang.Thread.State: WAITING (on object monitor)
  453. at java.lang.Object.wait(Native Method)
  454. at java.lang.Object.wait(Object.java:485)
  455. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  456. - locked <0x000000059add8a28> (a java.util.LinkedList)
  457. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  458. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  459. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  460. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  461. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  462. at java.lang.reflect.Method.invoke(Method.java:597)
  463. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  464. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  465. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  466. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  467. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  468. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  469. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  470. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  471. at java.lang.reflect.Method.invoke(Method.java:597)
  472. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  473. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  474.  
  475. "IPC Server handler 88 on 60020" daemon prio=10 tid=0x00007f983c523000 nid=0x1b6b in Object.wait() [0x00007f982eead000]
  476. java.lang.Thread.State: WAITING (on object monitor)
  477. at java.lang.Object.wait(Native Method)
  478. at java.lang.Object.wait(Object.java:485)
  479. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  480. - locked <0x0000000420e7bf98> (a java.util.TreeSet)
  481. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  482. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  483. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  484. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  485. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  486. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  487. at java.lang.reflect.Method.invoke(Method.java:597)
  488. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  489. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  490.  
  491. "IPC Server handler 87 on 60020" daemon prio=10 tid=0x00007f983c521000 nid=0x1b6a in Object.wait() [0x00007f982efae000]
  492. java.lang.Thread.State: WAITING (on object monitor)
  493. at java.lang.Object.wait(Native Method)
  494. at java.lang.Object.wait(Object.java:485)
  495. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  496. - locked <0x000000059add8a28> (a java.util.LinkedList)
  497. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  498. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  499. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  500. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  501. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  502. at java.lang.reflect.Method.invoke(Method.java:597)
  503. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  504. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  505. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  506. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  507. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  508. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  509. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  510. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  511. at java.lang.reflect.Method.invoke(Method.java:597)
  512. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  513. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  514.  
  515. "IPC Server handler 86 on 60020" daemon prio=10 tid=0x00007f983c51f000 nid=0x1b69 in Object.wait() [0x00007f982f0af000]
  516. java.lang.Thread.State: WAITING (on object monitor)
  517. at java.lang.Object.wait(Native Method)
  518. at java.lang.Object.wait(Object.java:485)
  519. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  520. - locked <0x000000059add8a28> (a java.util.LinkedList)
  521. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  522. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  523. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  524. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  525. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  526. at java.lang.reflect.Method.invoke(Method.java:597)
  527. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  528. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  529. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  530. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  531. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  532. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  533. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  534. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  535. at java.lang.reflect.Method.invoke(Method.java:597)
  536. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  537. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  538.  
  539. "IPC Server handler 85 on 60020" daemon prio=10 tid=0x00007f983c51d000 nid=0x1b68 in Object.wait() [0x00007f982f1b0000]
  540. java.lang.Thread.State: WAITING (on object monitor)
  541. at java.lang.Object.wait(Native Method)
  542. at java.lang.Object.wait(Object.java:485)
  543. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  544. - locked <0x0000000420ec2fa0> (a java.util.TreeSet)
  545. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  546. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  547. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  548. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  549. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  550. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  551. at java.lang.reflect.Method.invoke(Method.java:597)
  552. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  553. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  554.  
  555. "IPC Server handler 84 on 60020" daemon prio=10 tid=0x00007f983c51b000 nid=0x1b67 in Object.wait() [0x00007f982f2b1000]
  556. java.lang.Thread.State: WAITING (on object monitor)
  557. at java.lang.Object.wait(Native Method)
  558. at java.lang.Object.wait(Object.java:485)
  559. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  560. - locked <0x000000059add8a28> (a java.util.LinkedList)
  561. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  562. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  563. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  564. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  565. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  566. at java.lang.reflect.Method.invoke(Method.java:597)
  567. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  568. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  569. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  570. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  571. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  572. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  573. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  574. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  575. at java.lang.reflect.Method.invoke(Method.java:597)
  576. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  577. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  578.  
  579. "IPC Server handler 83 on 60020" daemon prio=10 tid=0x00007f983c519800 nid=0x1b66 in Object.wait() [0x00007f982f3b2000]
  580. java.lang.Thread.State: WAITING (on object monitor)
  581. at java.lang.Object.wait(Native Method)
  582. at java.lang.Object.wait(Object.java:485)
  583. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  584. - locked <0x000000041ee216a0> (a java.util.TreeSet)
  585. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  586. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  587. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  588. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  589. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  590. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  591. at java.lang.reflect.Method.invoke(Method.java:597)
  592. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  593. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  594.  
  595. "IPC Server handler 82 on 60020" daemon prio=10 tid=0x00007f983c517800 nid=0x1b65 in Object.wait() [0x00007f982f4b3000]
  596. java.lang.Thread.State: WAITING (on object monitor)
  597. at java.lang.Object.wait(Native Method)
  598. at java.lang.Object.wait(Object.java:485)
  599. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  600. - locked <0x000000059add8a28> (a java.util.LinkedList)
  601. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  602. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  603. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  604. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  605. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  606. at java.lang.reflect.Method.invoke(Method.java:597)
  607. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  608. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  609. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  610. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  611. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  612. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  613. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  614. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  615. at java.lang.reflect.Method.invoke(Method.java:597)
  616. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  617. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  618.  
  619. "IPC Server handler 81 on 60020" daemon prio=10 tid=0x00007f983c516000 nid=0x1b64 in Object.wait() [0x00007f982f5b4000]
  620. java.lang.Thread.State: WAITING (on object monitor)
  621. at java.lang.Object.wait(Native Method)
  622. at java.lang.Object.wait(Object.java:485)
  623. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  624. - locked <0x000000059add8a28> (a java.util.LinkedList)
  625. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  626. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  627. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  628. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  629. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  630. at java.lang.reflect.Method.invoke(Method.java:597)
  631. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  632. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  633. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  634. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  635. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  636. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  637. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  638. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  639. at java.lang.reflect.Method.invoke(Method.java:597)
  640. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  641. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  642.  
  643. "IPC Server handler 80 on 60020" daemon prio=10 tid=0x00007f983c514000 nid=0x1b63 in Object.wait() [0x00007f982f6b5000]
  644. java.lang.Thread.State: WAITING (on object monitor)
  645. at java.lang.Object.wait(Native Method)
  646. at java.lang.Object.wait(Object.java:485)
  647. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  648. - locked <0x000000042007c010> (a java.util.TreeSet)
  649. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  650. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  651. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  652. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  653. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  654. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  655. at java.lang.reflect.Method.invoke(Method.java:597)
  656. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  657. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  658.  
  659. "IPC Server handler 79 on 60020" daemon prio=10 tid=0x00007f983c512000 nid=0x1b62 in Object.wait() [0x00007f982f7b6000]
  660. java.lang.Thread.State: WAITING (on object monitor)
  661. at java.lang.Object.wait(Native Method)
  662. at java.lang.Object.wait(Object.java:485)
  663. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  664. - locked <0x000000041fe5c400> (a java.util.TreeSet)
  665. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  666. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  667. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  668. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  669. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  670. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  671. at java.lang.reflect.Method.invoke(Method.java:597)
  672. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  673. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  674.  
  675. "IPC Server handler 78 on 60020" daemon prio=10 tid=0x00007f983c510000 nid=0x1b61 in Object.wait() [0x00007f982f8b7000]
  676. java.lang.Thread.State: WAITING (on object monitor)
  677. at java.lang.Object.wait(Native Method)
  678. at java.lang.Object.wait(Object.java:485)
  679. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  680. - locked <0x000000059add8a28> (a java.util.LinkedList)
  681. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  682. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  683. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  684. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  685. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  686. at java.lang.reflect.Method.invoke(Method.java:597)
  687. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  688. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  689. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  690. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  691. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  692. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  693. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  694. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  695. at java.lang.reflect.Method.invoke(Method.java:597)
  696. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  697. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  698.  
  699. "IPC Server handler 77 on 60020" daemon prio=10 tid=0x00007f983c50e000 nid=0x1b60 in Object.wait() [0x00007f982f9b8000]
  700. java.lang.Thread.State: WAITING (on object monitor)
  701. at java.lang.Object.wait(Native Method)
  702. at java.lang.Object.wait(Object.java:485)
  703. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  704. - locked <0x00000004207a7988> (a java.util.TreeSet)
  705. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  706. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  707. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  708. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  709. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  710. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  711. at java.lang.reflect.Method.invoke(Method.java:597)
  712. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  713. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  714.  
  715. "IPC Server handler 76 on 60020" daemon prio=10 tid=0x00007f983c50c000 nid=0x1b5f in Object.wait() [0x00007f982fab9000]
  716. java.lang.Thread.State: WAITING (on object monitor)
  717. at java.lang.Object.wait(Native Method)
  718. at java.lang.Object.wait(Object.java:485)
  719. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  720. - locked <0x000000059add8a28> (a java.util.LinkedList)
  721. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  722. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  723. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  724. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  725. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  726. at java.lang.reflect.Method.invoke(Method.java:597)
  727. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  728. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  729. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  730. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  731. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  732. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  733. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  734. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  735. at java.lang.reflect.Method.invoke(Method.java:597)
  736. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  737. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  738.  
  739. "IPC Server handler 75 on 60020" daemon prio=10 tid=0x00007f983c50a000 nid=0x1b5e in Object.wait() [0x00007f982fbba000]
  740. java.lang.Thread.State: WAITING (on object monitor)
  741. at java.lang.Object.wait(Native Method)
  742. at java.lang.Object.wait(Object.java:485)
  743. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  744. - locked <0x000000059add8a28> (a java.util.LinkedList)
  745. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  746. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  747. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  748. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  749. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  750. at java.lang.reflect.Method.invoke(Method.java:597)
  751. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  752. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  753. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  754. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  755. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  756. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  757. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  758. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  759. at java.lang.reflect.Method.invoke(Method.java:597)
  760. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  761. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  762.  
  763. "IPC Server handler 74 on 60020" daemon prio=10 tid=0x00007f983c508000 nid=0x1b5d in Object.wait() [0x00007f982fcbb000]
  764. java.lang.Thread.State: WAITING (on object monitor)
  765. at java.lang.Object.wait(Native Method)
  766. at java.lang.Object.wait(Object.java:485)
  767. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  768. - locked <0x000000041fb4b3d0> (a java.util.TreeSet)
  769. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  770. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  771. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  772. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  773. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  774. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  775. at java.lang.reflect.Method.invoke(Method.java:597)
  776. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  777. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  778.  
  779. "IPC Server handler 73 on 60020" daemon prio=10 tid=0x00007f983c506000 nid=0x1b5c in Object.wait() [0x00007f982fdbc000]
  780. java.lang.Thread.State: WAITING (on object monitor)
  781. at java.lang.Object.wait(Native Method)
  782. at java.lang.Object.wait(Object.java:485)
  783. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  784. - locked <0x0000000420a61a90> (a java.util.TreeSet)
  785. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  786. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  787. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  788. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  789. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  790. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  791. at java.lang.reflect.Method.invoke(Method.java:597)
  792. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  793. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  794.  
  795. "IPC Server handler 72 on 60020" daemon prio=10 tid=0x00007f983c504000 nid=0x1b5b in Object.wait() [0x00007f982febd000]
  796. java.lang.Thread.State: WAITING (on object monitor)
  797. at java.lang.Object.wait(Native Method)
  798. at java.lang.Object.wait(Object.java:485)
  799. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  800. - locked <0x000000059add8a28> (a java.util.LinkedList)
  801. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  802. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  803. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  804. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  805. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  806. at java.lang.reflect.Method.invoke(Method.java:597)
  807. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  808. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  809. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  810. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  811. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  812. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  813. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  814. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  815. at java.lang.reflect.Method.invoke(Method.java:597)
  816. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  817. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  818.  
  819. "IPC Server handler 71 on 60020" daemon prio=10 tid=0x00007f983c502000 nid=0x1b5a in Object.wait() [0x00007f982ffbe000]
  820. java.lang.Thread.State: WAITING (on object monitor)
  821. at java.lang.Object.wait(Native Method)
  822. at java.lang.Object.wait(Object.java:485)
  823. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  824. - locked <0x000000059add8a28> (a java.util.LinkedList)
  825. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  826. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  827. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  828. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  829. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  830. at java.lang.reflect.Method.invoke(Method.java:597)
  831. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  832. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  833. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  834. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  835. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  836. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  837. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  838. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  839. at java.lang.reflect.Method.invoke(Method.java:597)
  840. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  841. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  842.  
  843. "IPC Server handler 70 on 60020" daemon prio=10 tid=0x00007f983c500000 nid=0x1b59 in Object.wait() [0x00007f98300bf000]
  844. java.lang.Thread.State: WAITING (on object monitor)
  845. at java.lang.Object.wait(Native Method)
  846. at java.lang.Object.wait(Object.java:485)
  847. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  848. - locked <0x000000041f3b7470> (a java.util.TreeSet)
  849. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  850. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1172)
  851. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  852. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  853. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  854. at java.lang.reflect.Method.invoke(Method.java:597)
  855. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  856. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  857.  
  858. "IPC Server handler 69 on 60020" daemon prio=10 tid=0x00007f983c4fe000 nid=0x1b58 in Object.wait() [0x00007f98301c0000]
  859. java.lang.Thread.State: WAITING (on object monitor)
  860. at java.lang.Object.wait(Native Method)
  861. at java.lang.Object.wait(Object.java:485)
  862. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  863. - locked <0x000000059add8a28> (a java.util.LinkedList)
  864. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  865. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  866. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  867. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  868. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  869. at java.lang.reflect.Method.invoke(Method.java:597)
  870. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  871. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  872. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  873. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  874. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  875. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  876. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  877. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  878. at java.lang.reflect.Method.invoke(Method.java:597)
  879. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  880. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  881.  
  882. "IPC Server handler 68 on 60020" daemon prio=10 tid=0x00007f983c4fc000 nid=0x1b57 in Object.wait() [0x00007f98302c1000]
  883. java.lang.Thread.State: WAITING (on object monitor)
  884. at java.lang.Object.wait(Native Method)
  885. at java.lang.Object.wait(Object.java:485)
  886. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  887. - locked <0x000000041fc61ff8> (a java.util.TreeSet)
  888. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  889. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  890. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  891. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  892. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  893. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  894. at java.lang.reflect.Method.invoke(Method.java:597)
  895. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  896. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  897.  
  898. "IPC Server handler 67 on 60020" daemon prio=10 tid=0x00007f983c4fa000 nid=0x1b56 in Object.wait() [0x00007f98303c2000]
  899. java.lang.Thread.State: WAITING (on object monitor)
  900. at java.lang.Object.wait(Native Method)
  901. at java.lang.Object.wait(Object.java:485)
  902. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  903. - locked <0x000000059add8a28> (a java.util.LinkedList)
  904. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  905. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  906. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  907. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  908. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  909. at java.lang.reflect.Method.invoke(Method.java:597)
  910. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  911. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  912. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  913. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  914. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  915. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  916. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  917. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  918. at java.lang.reflect.Method.invoke(Method.java:597)
  919. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  920. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  921.  
  922. "IPC Server handler 66 on 60020" daemon prio=10 tid=0x00007f983c4f8000 nid=0x1b55 in Object.wait() [0x00007f98304c3000]
  923. java.lang.Thread.State: WAITING (on object monitor)
  924. at java.lang.Object.wait(Native Method)
  925. at java.lang.Object.wait(Object.java:485)
  926. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  927. - locked <0x000000059add8a28> (a java.util.LinkedList)
  928. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  929. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  930. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  931. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  932. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  933. at java.lang.reflect.Method.invoke(Method.java:597)
  934. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  935. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  936. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  937. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  938. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  939. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  940. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  941. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  942. at java.lang.reflect.Method.invoke(Method.java:597)
  943. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  944. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  945.  
  946. "IPC Server handler 65 on 60020" daemon prio=10 tid=0x00007f983c4f6000 nid=0x1b54 in Object.wait() [0x00007f98305c4000]
  947. java.lang.Thread.State: WAITING (on object monitor)
  948. at java.lang.Object.wait(Native Method)
  949. at java.lang.Object.wait(Object.java:485)
  950. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  951. - locked <0x000000059add8a28> (a java.util.LinkedList)
  952. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  953. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  954. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  955. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  956. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  957. at java.lang.reflect.Method.invoke(Method.java:597)
  958. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  959. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  960. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  961. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  962. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  963. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  964. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  965. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  966. at java.lang.reflect.Method.invoke(Method.java:597)
  967. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  968. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  969.  
  970. "IPC Server handler 64 on 60020" daemon prio=10 tid=0x00007f983c4f4000 nid=0x1b53 in Object.wait() [0x00007f98306c5000]
  971. java.lang.Thread.State: WAITING (on object monitor)
  972. at java.lang.Object.wait(Native Method)
  973. at java.lang.Object.wait(Object.java:485)
  974. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  975. - locked <0x0000000420ec2fa0> (a java.util.TreeSet)
  976. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  977. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  978. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  979. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  980. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  981. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  982. at java.lang.reflect.Method.invoke(Method.java:597)
  983. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  984. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  985.  
  986. "IPC Server handler 63 on 60020" daemon prio=10 tid=0x00007f983c4f2000 nid=0x1b52 in Object.wait() [0x00007f98307c6000]
  987. java.lang.Thread.State: WAITING (on object monitor)
  988. at java.lang.Object.wait(Native Method)
  989. at java.lang.Object.wait(Object.java:485)
  990. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  991. - locked <0x000000059add8a28> (a java.util.LinkedList)
  992. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  993. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  994. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  995. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  996. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  997. at java.lang.reflect.Method.invoke(Method.java:597)
  998. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  999. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1000. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1001. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1002. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1003. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1004. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1005. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1006. at java.lang.reflect.Method.invoke(Method.java:597)
  1007. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1008. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1009.  
  1010. "IPC Server handler 62 on 60020" daemon prio=10 tid=0x00007f983c4f0000 nid=0x1b51 in Object.wait() [0x00007f98308c7000]
  1011. java.lang.Thread.State: WAITING (on object monitor)
  1012. at java.lang.Object.wait(Native Method)
  1013. at java.lang.Object.wait(Object.java:485)
  1014. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1015. - locked <0x00000004206e7390> (a java.util.TreeSet)
  1016. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1017. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  1018. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1019. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1020. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1021. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1022. at java.lang.reflect.Method.invoke(Method.java:597)
  1023. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1024. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1025.  
  1026. "IPC Server handler 61 on 60020" daemon prio=10 tid=0x00007f983c4ee000 nid=0x1b50 in Object.wait() [0x00007f98309c8000]
  1027. java.lang.Thread.State: WAITING (on object monitor)
  1028. at java.lang.Object.wait(Native Method)
  1029. at java.lang.Object.wait(Object.java:485)
  1030. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1031. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1032. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1033. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1034. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1035. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1036. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1037. at java.lang.reflect.Method.invoke(Method.java:597)
  1038. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1039. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1040. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1041. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  1042. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  1043. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1044. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1045. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1046. at java.lang.reflect.Method.invoke(Method.java:597)
  1047. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1048. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1049.  
  1050. "IPC Server handler 60 on 60020" daemon prio=10 tid=0x00007f983c4ec800 nid=0x1b4f in Object.wait() [0x00007f9830ac9000]
  1051. java.lang.Thread.State: WAITING (on object monitor)
  1052. at java.lang.Object.wait(Native Method)
  1053. at java.lang.Object.wait(Object.java:485)
  1054. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1055. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1056. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1057. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1058. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1059. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1060. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1061. at java.lang.reflect.Method.invoke(Method.java:597)
  1062. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1063. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1064. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1065. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1066. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1067. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1068. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1069. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1070. at java.lang.reflect.Method.invoke(Method.java:597)
  1071. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1072. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1073.  
  1074. "IPC Server handler 59 on 60020" daemon prio=10 tid=0x00007f983c4eb000 nid=0x1b4e in Object.wait() [0x00007f9830bca000]
  1075. java.lang.Thread.State: WAITING (on object monitor)
  1076. at java.lang.Object.wait(Native Method)
  1077. at java.lang.Object.wait(Object.java:485)
  1078. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1079. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1080. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1081. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1082. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1083. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1084. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1085. at java.lang.reflect.Method.invoke(Method.java:597)
  1086. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1087. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1088. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1089. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1090. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1091. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1092. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1093. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1094. at java.lang.reflect.Method.invoke(Method.java:597)
  1095. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1096. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1097.  
  1098. "IPC Server handler 58 on 60020" daemon prio=10 tid=0x00007f983c4e9000 nid=0x1b4d in Object.wait() [0x00007f9830ccb000]
  1099. java.lang.Thread.State: WAITING (on object monitor)
  1100. at java.lang.Object.wait(Native Method)
  1101. at java.lang.Object.wait(Object.java:485)
  1102. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1103. - locked <0x0000000420731890> (a java.util.TreeSet)
  1104. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1105. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1172)
  1106. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1107. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1108. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1109. at java.lang.reflect.Method.invoke(Method.java:597)
  1110. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1111. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1112.  
  1113. "IPC Server handler 57 on 60020" daemon prio=10 tid=0x00007f983c4e7000 nid=0x1b4c in Object.wait() [0x00007f9830dcc000]
  1114. java.lang.Thread.State: WAITING (on object monitor)
  1115. at java.lang.Object.wait(Native Method)
  1116. at java.lang.Object.wait(Object.java:485)
  1117. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1118. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1119. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1120. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1121. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1122. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1123. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1124. at java.lang.reflect.Method.invoke(Method.java:597)
  1125. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1126. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1127. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1128. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  1129. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  1130. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1131. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1132. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1133. at java.lang.reflect.Method.invoke(Method.java:597)
  1134. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1135. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1136.  
  1137. "IPC Server handler 56 on 60020" daemon prio=10 tid=0x00007f983c4e6800 nid=0x1b4b in Object.wait() [0x00007f9830ecd000]
  1138. java.lang.Thread.State: WAITING (on object monitor)
  1139. at java.lang.Object.wait(Native Method)
  1140. at java.lang.Object.wait(Object.java:485)
  1141. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1142. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1143. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1144. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1145. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1146. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1147. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1148. at java.lang.reflect.Method.invoke(Method.java:597)
  1149. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1150. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1151. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1152. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  1153. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  1154. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1155. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1156. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1157. at java.lang.reflect.Method.invoke(Method.java:597)
  1158. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1159. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1160.  
  1161. "IPC Server handler 55 on 60020" daemon prio=10 tid=0x00007f983c4e4800 nid=0x1b4a in Object.wait() [0x00007f9830fce000]
  1162. java.lang.Thread.State: WAITING (on object monitor)
  1163. at java.lang.Object.wait(Native Method)
  1164. at java.lang.Object.wait(Object.java:485)
  1165. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1166. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1167. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1168. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1169. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1170. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1171. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1172. at java.lang.reflect.Method.invoke(Method.java:597)
  1173. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1174. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1175. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1176. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1177. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1178. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1179. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1180. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1181. at java.lang.reflect.Method.invoke(Method.java:597)
  1182. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1183. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1184.  
  1185. "IPC Server handler 54 on 60020" daemon prio=10 tid=0x00007f983c4e2800 nid=0x1b49 in Object.wait() [0x00007f98310cf000]
  1186. java.lang.Thread.State: WAITING (on object monitor)
  1187. at java.lang.Object.wait(Native Method)
  1188. at java.lang.Object.wait(Object.java:485)
  1189. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1190. - locked <0x00000004237c8718> (a java.util.TreeSet)
  1191. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1192. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  1193. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1194. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1195. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1196. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1197. at java.lang.reflect.Method.invoke(Method.java:597)
  1198. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1199. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1200.  
  1201. "IPC Server handler 53 on 60020" daemon prio=10 tid=0x00007f983c4e0800 nid=0x1b48 in Object.wait() [0x00007f98311d0000]
  1202. java.lang.Thread.State: WAITING (on object monitor)
  1203. at java.lang.Object.wait(Native Method)
  1204. at java.lang.Object.wait(Object.java:485)
  1205. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1206. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1207. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1208. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1209. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1210. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1211. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1212. at java.lang.reflect.Method.invoke(Method.java:597)
  1213. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1214. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1215. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1216. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1217. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1218. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1219. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1220. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1221. at java.lang.reflect.Method.invoke(Method.java:597)
  1222. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1223. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1224.  
  1225. "IPC Server handler 52 on 60020" daemon prio=10 tid=0x00007f983c4cf800 nid=0x1b47 in Object.wait() [0x00007f98312d1000]
  1226. java.lang.Thread.State: WAITING (on object monitor)
  1227. at java.lang.Object.wait(Native Method)
  1228. at java.lang.Object.wait(Object.java:485)
  1229. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1230. - locked <0x0000000420043e38> (a java.util.TreeSet)
  1231. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1232. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1172)
  1233. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1234. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1235. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1236. at java.lang.reflect.Method.invoke(Method.java:597)
  1237. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1238. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1239.  
  1240. "IPC Server handler 51 on 60020" daemon prio=10 tid=0x00007f983c4cd800 nid=0x1b46 in Object.wait() [0x00007f98313d2000]
  1241. java.lang.Thread.State: WAITING (on object monitor)
  1242. at java.lang.Object.wait(Native Method)
  1243. at java.lang.Object.wait(Object.java:485)
  1244. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1245. - locked <0x00000004203312e0> (a java.util.TreeSet)
  1246. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1247. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  1248. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1249. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1250. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1251. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1252. at java.lang.reflect.Method.invoke(Method.java:597)
  1253. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1254. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1255.  
  1256. "IPC Server handler 50 on 60020" daemon prio=10 tid=0x00007f983c4cc000 nid=0x1b45 in Object.wait() [0x00007f98314d3000]
  1257. java.lang.Thread.State: WAITING (on object monitor)
  1258. at java.lang.Object.wait(Native Method)
  1259. at java.lang.Object.wait(Object.java:485)
  1260. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1261. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1262. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1263. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1264. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1265. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1266. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1267. at java.lang.reflect.Method.invoke(Method.java:597)
  1268. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1269. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1270. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1271. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  1272. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  1273. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1274. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1275. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1276. at java.lang.reflect.Method.invoke(Method.java:597)
  1277. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1278. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1279.  
  1280. "IPC Server handler 49 on 60020" daemon prio=10 tid=0x00007f983c4ca000 nid=0x1b44 in Object.wait() [0x00007f98315d4000]
  1281. java.lang.Thread.State: WAITING (on object monitor)
  1282. at java.lang.Object.wait(Native Method)
  1283. at java.lang.Object.wait(Object.java:485)
  1284. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1285. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1286. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1287. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1288. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1289. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1290. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1291. at java.lang.reflect.Method.invoke(Method.java:597)
  1292. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1293. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1294. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1295. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1296. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1297. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1298. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1299. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1300. at java.lang.reflect.Method.invoke(Method.java:597)
  1301. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1302. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1303.  
  1304. "IPC Server handler 48 on 60020" daemon prio=10 tid=0x00007f983c4c8000 nid=0x1b43 in Object.wait() [0x00007f98316d5000]
  1305. java.lang.Thread.State: WAITING (on object monitor)
  1306. at java.lang.Object.wait(Native Method)
  1307. at java.lang.Object.wait(Object.java:485)
  1308. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1309. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1310. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1311. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1312. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1313. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1314. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1315. at java.lang.reflect.Method.invoke(Method.java:597)
  1316. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1317. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1318. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1319. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1320. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1321. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1322. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1323. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1324. at java.lang.reflect.Method.invoke(Method.java:597)
  1325. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1326. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1327.  
  1328. "IPC Server handler 47 on 60020" daemon prio=10 tid=0x00007f983c4c6000 nid=0x1b42 in Object.wait() [0x00007f98317d6000]
  1329. java.lang.Thread.State: WAITING (on object monitor)
  1330. at java.lang.Object.wait(Native Method)
  1331. at java.lang.Object.wait(Object.java:485)
  1332. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1333. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1334. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1335. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1336. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1337. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1338. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1339. at java.lang.reflect.Method.invoke(Method.java:597)
  1340. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1341. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1342. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1343. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1344. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1345. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1346. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1347. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1348. at java.lang.reflect.Method.invoke(Method.java:597)
  1349. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1350. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1351.  
  1352. "IPC Server handler 46 on 60020" daemon prio=10 tid=0x00007f983c601000 nid=0x1b41 in Object.wait() [0x00007f98318d7000]
  1353. java.lang.Thread.State: WAITING (on object monitor)
  1354. at java.lang.Object.wait(Native Method)
  1355. at java.lang.Object.wait(Object.java:485)
  1356. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1357. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1358. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1359. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1360. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1361. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1362. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1363. at java.lang.reflect.Method.invoke(Method.java:597)
  1364. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1365. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1366. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1367. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1368. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1369. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1370. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1371. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1372. at java.lang.reflect.Method.invoke(Method.java:597)
  1373. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1374. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1375.  
  1376. "IPC Server handler 45 on 60020" daemon prio=10 tid=0x00007f983c5fe800 nid=0x1b40 in Object.wait() [0x00007f98319d8000]
  1377. java.lang.Thread.State: WAITING (on object monitor)
  1378. at java.lang.Object.wait(Native Method)
  1379. at java.lang.Object.wait(Object.java:485)
  1380. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1381. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1382. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1383. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1384. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1385. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1386. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1387. at java.lang.reflect.Method.invoke(Method.java:597)
  1388. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1389. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1390. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1391. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1392. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1393. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1394. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1395. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1396. at java.lang.reflect.Method.invoke(Method.java:597)
  1397. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1398. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1399.  
  1400. "IPC Server handler 44 on 60020" daemon prio=10 tid=0x00007f983c5fc800 nid=0x1b3f in Object.wait() [0x00007f9831ad9000]
  1401. java.lang.Thread.State: WAITING (on object monitor)
  1402. at java.lang.Object.wait(Native Method)
  1403. at java.lang.Object.wait(Object.java:485)
  1404. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1405. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1406. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1407. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1408. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1409. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1410. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1411. at java.lang.reflect.Method.invoke(Method.java:597)
  1412. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1413. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1414. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1415. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1416. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1417. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1418. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1419. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1420. at java.lang.reflect.Method.invoke(Method.java:597)
  1421. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1422. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1423.  
  1424. "IPC Server handler 43 on 60020" daemon prio=10 tid=0x00007f983c5fa800 nid=0x1b3e in Object.wait() [0x00007f9831bda000]
  1425. java.lang.Thread.State: WAITING (on object monitor)
  1426. at java.lang.Object.wait(Native Method)
  1427. at java.lang.Object.wait(Object.java:485)
  1428. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1429. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1430. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1431. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1432. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1433. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1434. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1435. at java.lang.reflect.Method.invoke(Method.java:597)
  1436. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1437. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1438. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1439. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1440. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1441. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1442. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1443. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1444. at java.lang.reflect.Method.invoke(Method.java:597)
  1445. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1446. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1447.  
  1448. "IPC Server handler 42 on 60020" daemon prio=10 tid=0x00007f983c5f8800 nid=0x1b3d in Object.wait() [0x00007f9831cdb000]
  1449. java.lang.Thread.State: WAITING (on object monitor)
  1450. at java.lang.Object.wait(Native Method)
  1451. at java.lang.Object.wait(Object.java:485)
  1452. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1453. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1454. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1455. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1456. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1457. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1458. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1459. at java.lang.reflect.Method.invoke(Method.java:597)
  1460. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1461. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1462. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1463. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1464. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1465. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1466. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1467. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1468. at java.lang.reflect.Method.invoke(Method.java:597)
  1469. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1470. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1471.  
  1472. "IPC Server handler 41 on 60020" daemon prio=10 tid=0x00007f983c5f6800 nid=0x1b3c in Object.wait() [0x00007f9831ddc000]
  1473. java.lang.Thread.State: WAITING (on object monitor)
  1474. at java.lang.Object.wait(Native Method)
  1475. at java.lang.Object.wait(Object.java:485)
  1476. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1477. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1478. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1479. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1480. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1481. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1482. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1483. at java.lang.reflect.Method.invoke(Method.java:597)
  1484. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1485. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1486. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1487. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1488. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1489. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1490. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1491. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1492. at java.lang.reflect.Method.invoke(Method.java:597)
  1493. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1494. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1495.  
  1496. "IPC Server handler 40 on 60020" daemon prio=10 tid=0x00007f983c5f5000 nid=0x1b3b in Object.wait() [0x00007f9831edd000]
  1497. java.lang.Thread.State: WAITING (on object monitor)
  1498. at java.lang.Object.wait(Native Method)
  1499. at java.lang.Object.wait(Object.java:485)
  1500. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1501. - locked <0x0000000431b1fed8> (a java.util.TreeSet)
  1502. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1503. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  1504. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1505. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1506. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1507. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1508. at java.lang.reflect.Method.invoke(Method.java:597)
  1509. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1510. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1511.  
  1512. "IPC Server handler 39 on 60020" daemon prio=10 tid=0x00007f983c5f3000 nid=0x1b3a in Object.wait() [0x00007f9831fde000]
  1513. java.lang.Thread.State: WAITING (on object monitor)
  1514. at java.lang.Object.wait(Native Method)
  1515. at java.lang.Object.wait(Object.java:485)
  1516. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1517. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1518. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1519. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1520. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1521. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1522. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1523. at java.lang.reflect.Method.invoke(Method.java:597)
  1524. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1525. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1526. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1527. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1528. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1529. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1530. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1531. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1532. at java.lang.reflect.Method.invoke(Method.java:597)
  1533. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1534. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1535.  
  1536. "IPC Server handler 38 on 60020" daemon prio=10 tid=0x00007f983c5f1000 nid=0x1b39 in Object.wait() [0x00007f98320df000]
  1537. java.lang.Thread.State: WAITING (on object monitor)
  1538. at java.lang.Object.wait(Native Method)
  1539. at java.lang.Object.wait(Object.java:485)
  1540. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1541. - locked <0x0000000420ec2fa0> (a java.util.TreeSet)
  1542. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1543. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  1544. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1545. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1546. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1547. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1548. at java.lang.reflect.Method.invoke(Method.java:597)
  1549. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1550. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1551.  
  1552. "IPC Server handler 37 on 60020" daemon prio=10 tid=0x00007f983c5ef000 nid=0x1b38 in Object.wait() [0x00007f98321e0000]
  1553. java.lang.Thread.State: WAITING (on object monitor)
  1554. at java.lang.Object.wait(Native Method)
  1555. at java.lang.Object.wait(Object.java:485)
  1556. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1557. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1558. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1559. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1560. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1561. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1562. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1563. at java.lang.reflect.Method.invoke(Method.java:597)
  1564. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1565. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1566. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1567. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1568. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1569. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1570. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1571. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1572. at java.lang.reflect.Method.invoke(Method.java:597)
  1573. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1574. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1575.  
  1576. "IPC Server handler 36 on 60020" daemon prio=10 tid=0x00007f983c5ed000 nid=0x1b37 in Object.wait() [0x00007f98322e1000]
  1577. java.lang.Thread.State: WAITING (on object monitor)
  1578. at java.lang.Object.wait(Native Method)
  1579. at java.lang.Object.wait(Object.java:485)
  1580. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1581. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1582. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1583. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1584. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1585. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1586. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1587. at java.lang.reflect.Method.invoke(Method.java:597)
  1588. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1589. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1590. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1591. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1592. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1593. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1594. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1595. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1596. at java.lang.reflect.Method.invoke(Method.java:597)
  1597. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1598. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1599.  
  1600. "IPC Server handler 35 on 60020" daemon prio=10 tid=0x00007f983c5eb000 nid=0x1b36 in Object.wait() [0x00007f98323e2000]
  1601. java.lang.Thread.State: WAITING (on object monitor)
  1602. at java.lang.Object.wait(Native Method)
  1603. at java.lang.Object.wait(Object.java:485)
  1604. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1605. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1606. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1607. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1608. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1609. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1610. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1611. at java.lang.reflect.Method.invoke(Method.java:597)
  1612. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1613. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1614. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1615. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1616. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1617. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1618. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1619. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1620. at java.lang.reflect.Method.invoke(Method.java:597)
  1621. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1622. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1623.  
  1624. "IPC Server handler 34 on 60020" daemon prio=10 tid=0x00007f983c5e9000 nid=0x1b35 in Object.wait() [0x00007f98324e3000]
  1625. java.lang.Thread.State: WAITING (on object monitor)
  1626. at java.lang.Object.wait(Native Method)
  1627. at java.lang.Object.wait(Object.java:485)
  1628. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1629. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1630. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1631. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1632. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1633. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1634. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1635. at java.lang.reflect.Method.invoke(Method.java:597)
  1636. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1637. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1638. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1639. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1640. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1641. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1642. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1643. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1644. at java.lang.reflect.Method.invoke(Method.java:597)
  1645. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1646. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1647.  
  1648. "IPC Server handler 33 on 60020" daemon prio=10 tid=0x00007f983c5e7000 nid=0x1b34 in Object.wait() [0x00007f98325e4000]
  1649. java.lang.Thread.State: WAITING (on object monitor)
  1650. at java.lang.Object.wait(Native Method)
  1651. at java.lang.Object.wait(Object.java:485)
  1652. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1653. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1654. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1655. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1656. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1657. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1658. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1659. at java.lang.reflect.Method.invoke(Method.java:597)
  1660. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1661. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1662. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1663. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  1664. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  1665. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1666. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1667. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1668. at java.lang.reflect.Method.invoke(Method.java:597)
  1669. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1670. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1671.  
  1672. "IPC Server handler 32 on 60020" daemon prio=10 tid=0x00007f983c5e5000 nid=0x1b33 in Object.wait() [0x00007f98326e5000]
  1673. java.lang.Thread.State: WAITING (on object monitor)
  1674. at java.lang.Object.wait(Native Method)
  1675. at java.lang.Object.wait(Object.java:485)
  1676. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1677. - locked <0x000000041e9d0428> (a java.util.TreeSet)
  1678. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1679. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1172)
  1680. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1681. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1682. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1683. at java.lang.reflect.Method.invoke(Method.java:597)
  1684. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1685. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1686.  
  1687. "IPC Server handler 31 on 60020" daemon prio=10 tid=0x00007f983c5e3000 nid=0x1b32 in Object.wait() [0x00007f98327e6000]
  1688. java.lang.Thread.State: WAITING (on object monitor)
  1689. at java.lang.Object.wait(Native Method)
  1690. at java.lang.Object.wait(Object.java:485)
  1691. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1692. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1693. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1694. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1695. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1696. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1697. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1698. at java.lang.reflect.Method.invoke(Method.java:597)
  1699. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1700. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1701. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1702. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1703. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1704. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1705. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1706. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1707. at java.lang.reflect.Method.invoke(Method.java:597)
  1708. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1709. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1710.  
  1711. "IPC Server handler 30 on 60020" daemon prio=10 tid=0x00007f983c5e1000 nid=0x1b31 in Object.wait() [0x00007f98328e7000]
  1712. java.lang.Thread.State: WAITING (on object monitor)
  1713. at java.lang.Object.wait(Native Method)
  1714. at java.lang.Object.wait(Object.java:485)
  1715. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1716. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1717. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1718. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1719. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1720. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1721. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1722. at java.lang.reflect.Method.invoke(Method.java:597)
  1723. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1724. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1725. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1726. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1727. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1728. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1729. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1730. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1731. at java.lang.reflect.Method.invoke(Method.java:597)
  1732. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1733. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1734.  
  1735. "IPC Server handler 29 on 60020" daemon prio=10 tid=0x00007f983c5df000 nid=0x1b30 in Object.wait() [0x00007f98329e8000]
  1736. java.lang.Thread.State: WAITING (on object monitor)
  1737. at java.lang.Object.wait(Native Method)
  1738. at java.lang.Object.wait(Object.java:485)
  1739. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1740. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1741. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1742. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1743. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1744. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1745. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1746. at java.lang.reflect.Method.invoke(Method.java:597)
  1747. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1748. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1749. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1750. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1751. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1752. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1753. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1754. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1755. at java.lang.reflect.Method.invoke(Method.java:597)
  1756. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1757. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1758.  
  1759. "IPC Server handler 28 on 60020" daemon prio=10 tid=0x00007f983c5dd000 nid=0x1b2f in Object.wait() [0x00007f9832ae9000]
  1760. java.lang.Thread.State: WAITING (on object monitor)
  1761. at java.lang.Object.wait(Native Method)
  1762. at java.lang.Object.wait(Object.java:485)
  1763. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1764. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1765. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1766. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1767. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1768. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1769. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1770. at java.lang.reflect.Method.invoke(Method.java:597)
  1771. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1772. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1773. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1774. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1775. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1776. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1777. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1778. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1779. at java.lang.reflect.Method.invoke(Method.java:597)
  1780. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1781. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1782.  
  1783. "IPC Server handler 27 on 60020" daemon prio=10 tid=0x00007f983c5db000 nid=0x1b2e in Object.wait() [0x00007f9832bea000]
  1784. java.lang.Thread.State: WAITING (on object monitor)
  1785. at java.lang.Object.wait(Native Method)
  1786. at java.lang.Object.wait(Object.java:485)
  1787. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1788. - locked <0x0000000420bc1238> (a java.util.TreeSet)
  1789. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1790. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  1791. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1792. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1793. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1794. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1795. at java.lang.reflect.Method.invoke(Method.java:597)
  1796. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1797. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1798.  
  1799. "IPC Server handler 26 on 60020" daemon prio=10 tid=0x00007f983c5d9000 nid=0x1b2d in Object.wait() [0x00007f9832ceb000]
  1800. java.lang.Thread.State: WAITING (on object monitor)
  1801. at java.lang.Object.wait(Native Method)
  1802. at java.lang.Object.wait(Object.java:485)
  1803. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1804. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1805. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1806. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1807. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1808. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1809. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1810. at java.lang.reflect.Method.invoke(Method.java:597)
  1811. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1812. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1813. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1814. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1815. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1816. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1817. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1818. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1819. at java.lang.reflect.Method.invoke(Method.java:597)
  1820. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1821. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1822.  
  1823. "IPC Server handler 25 on 60020" daemon prio=10 tid=0x00007f983c5d7000 nid=0x1b2c in Object.wait() [0x00007f9832dec000]
  1824. java.lang.Thread.State: WAITING (on object monitor)
  1825. at java.lang.Object.wait(Native Method)
  1826. at java.lang.Object.wait(Object.java:485)
  1827. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1828. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1829. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1830. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1831. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1832. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1833. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1834. at java.lang.reflect.Method.invoke(Method.java:597)
  1835. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1836. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1837. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1838. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1839. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1840. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1841. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1842. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1843. at java.lang.reflect.Method.invoke(Method.java:597)
  1844. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1845. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1846.  
  1847. "IPC Server handler 24 on 60020" daemon prio=10 tid=0x00007f983c5d5000 nid=0x1b2b in Object.wait() [0x00007f9832eed000]
  1848. java.lang.Thread.State: WAITING (on object monitor)
  1849. at java.lang.Object.wait(Native Method)
  1850. at java.lang.Object.wait(Object.java:485)
  1851. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1852. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1853. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1854. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1855. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1856. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1857. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1858. at java.lang.reflect.Method.invoke(Method.java:597)
  1859. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1860. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1861. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1862. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1863. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1864. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1865. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1866. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1867. at java.lang.reflect.Method.invoke(Method.java:597)
  1868. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1869. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1870.  
  1871. "IPC Server handler 23 on 60020" daemon prio=10 tid=0x00007f983c5d3000 nid=0x1b2a in Object.wait() [0x00007f9832fee000]
  1872. java.lang.Thread.State: WAITING (on object monitor)
  1873. at java.lang.Object.wait(Native Method)
  1874. at java.lang.Object.wait(Object.java:485)
  1875. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  1876. - locked <0x000000041f49c2f0> (a java.util.TreeSet)
  1877. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  1878. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  1879. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1880. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1881. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1882. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1883. at java.lang.reflect.Method.invoke(Method.java:597)
  1884. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1885. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1886.  
  1887. "IPC Server handler 22 on 60020" daemon prio=10 tid=0x00007f983c5d1000 nid=0x1b29 in Object.wait() [0x00007f98330ef000]
  1888. java.lang.Thread.State: WAITING (on object monitor)
  1889. at java.lang.Object.wait(Native Method)
  1890. at java.lang.Object.wait(Object.java:485)
  1891. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1892. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1893. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1894. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1895. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1896. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1897. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1898. at java.lang.reflect.Method.invoke(Method.java:597)
  1899. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1900. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1901. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1902. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1903. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1904. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1905. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1906. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1907. at java.lang.reflect.Method.invoke(Method.java:597)
  1908. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1909. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1910.  
  1911. "IPC Server handler 21 on 60020" daemon prio=10 tid=0x00007f983c5cf000 nid=0x1b28 in Object.wait() [0x00007f98331f0000]
  1912. java.lang.Thread.State: WAITING (on object monitor)
  1913. at java.lang.Object.wait(Native Method)
  1914. at java.lang.Object.wait(Object.java:485)
  1915. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1916. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1917. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1918. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1919. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1920. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1921. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1922. at java.lang.reflect.Method.invoke(Method.java:597)
  1923. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1924. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1925. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1926. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1927. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1928. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1929. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1930. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1931. at java.lang.reflect.Method.invoke(Method.java:597)
  1932. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1933. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1934.  
  1935. "IPC Server handler 20 on 60020" daemon prio=10 tid=0x00007f983c5cd000 nid=0x1b27 in Object.wait() [0x00007f98332f1000]
  1936. java.lang.Thread.State: WAITING (on object monitor)
  1937. at java.lang.Object.wait(Native Method)
  1938. at java.lang.Object.wait(Object.java:485)
  1939. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1940. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1941. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1942. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1943. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1944. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1945. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1946. at java.lang.reflect.Method.invoke(Method.java:597)
  1947. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1948. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1949. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1950. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  1951. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  1952. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  1953. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  1954. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1955. at java.lang.reflect.Method.invoke(Method.java:597)
  1956. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1957. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1958.  
  1959. "IPC Server handler 19 on 60020" daemon prio=10 tid=0x00007f983c5cb000 nid=0x1b26 in Object.wait() [0x00007f98333f2000]
  1960. java.lang.Thread.State: WAITING (on object monitor)
  1961. at java.lang.Object.wait(Native Method)
  1962. at java.lang.Object.wait(Object.java:485)
  1963. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1964. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1965. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1966. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1967. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1968. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1969. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1970. at java.lang.reflect.Method.invoke(Method.java:597)
  1971. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1972. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1973. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1974. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1975. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  1976. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  1977. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  1978. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1979. at java.lang.reflect.Method.invoke(Method.java:597)
  1980. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  1981. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  1982.  
  1983. "IPC Server handler 18 on 60020" daemon prio=10 tid=0x00007f983c5c9000 nid=0x1b25 in Object.wait() [0x00007f98334f3000]
  1984. java.lang.Thread.State: WAITING (on object monitor)
  1985. at java.lang.Object.wait(Native Method)
  1986. at java.lang.Object.wait(Object.java:485)
  1987. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  1988. - locked <0x000000059add8a28> (a java.util.LinkedList)
  1989. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  1990. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  1991. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  1992. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  1993. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  1994. at java.lang.reflect.Method.invoke(Method.java:597)
  1995. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  1996. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  1997. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  1998. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  1999. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2000. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2001. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2002. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2003. at java.lang.reflect.Method.invoke(Method.java:597)
  2004. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2005. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2006.  
  2007. "IPC Server handler 17 on 60020" daemon prio=10 tid=0x00007f983c5c7000 nid=0x1b24 in Object.wait() [0x00007f98335f4000]
  2008. java.lang.Thread.State: WAITING (on object monitor)
  2009. at java.lang.Object.wait(Native Method)
  2010. at java.lang.Object.wait(Object.java:485)
  2011. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  2012. - locked <0x000000041e9a18e0> (a java.util.TreeSet)
  2013. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  2014. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  2015. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2016. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2017. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2018. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2019. at java.lang.reflect.Method.invoke(Method.java:597)
  2020. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2021. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2022.  
  2023. "IPC Server handler 16 on 60020" daemon prio=10 tid=0x00007f983c5c5000 nid=0x1b23 in Object.wait() [0x00007f98336f5000]
  2024. java.lang.Thread.State: WAITING (on object monitor)
  2025. at java.lang.Object.wait(Native Method)
  2026. at java.lang.Object.wait(Object.java:485)
  2027. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  2028. - locked <0x0000000420e7bf98> (a java.util.TreeSet)
  2029. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  2030. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  2031. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2032. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2033. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2034. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2035. at java.lang.reflect.Method.invoke(Method.java:597)
  2036. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2037. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2038.  
  2039. "IPC Server handler 15 on 60020" daemon prio=10 tid=0x00007f983c5c3000 nid=0x1b22 in Object.wait() [0x00007f98337f6000]
  2040. java.lang.Thread.State: WAITING (on object monitor)
  2041. at java.lang.Object.wait(Native Method)
  2042. at java.lang.Object.wait(Object.java:485)
  2043. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2044. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2045. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2046. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2047. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2048. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2049. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2050. at java.lang.reflect.Method.invoke(Method.java:597)
  2051. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2052. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2053. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2054. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2055. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2056. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2057. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2058. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2059. at java.lang.reflect.Method.invoke(Method.java:597)
  2060. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2061. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2062.  
  2063. "IPC Server handler 14 on 60020" daemon prio=10 tid=0x00007f983c5c1000 nid=0x1b21 in Object.wait() [0x00007f98338f7000]
  2064. java.lang.Thread.State: WAITING (on object monitor)
  2065. at java.lang.Object.wait(Native Method)
  2066. at java.lang.Object.wait(Object.java:485)
  2067. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2068. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2069. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2070. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2071. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2072. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2073. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2074. at java.lang.reflect.Method.invoke(Method.java:597)
  2075. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2076. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2077. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2078. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2079. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2080. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2081. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2082. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2083. at java.lang.reflect.Method.invoke(Method.java:597)
  2084. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2085. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2086.  
  2087. "IPC Server handler 13 on 60020" daemon prio=10 tid=0x00007f983c5bf000 nid=0x1b20 in Object.wait() [0x00007f98339f8000]
  2088. java.lang.Thread.State: WAITING (on object monitor)
  2089. at java.lang.Object.wait(Native Method)
  2090. at java.lang.Object.wait(Object.java:485)
  2091. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2092. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2093. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2094. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2095. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2096. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2097. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2098. at java.lang.reflect.Method.invoke(Method.java:597)
  2099. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2100. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2101. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2102. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2103. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2104. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2105. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2106. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2107. at java.lang.reflect.Method.invoke(Method.java:597)
  2108. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2109. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2110.  
  2111. "IPC Server handler 12 on 60020" daemon prio=10 tid=0x00007f983c5bd000 nid=0x1b1f in Object.wait() [0x00007f9833af9000]
  2112. java.lang.Thread.State: WAITING (on object monitor)
  2113. at java.lang.Object.wait(Native Method)
  2114. at java.lang.Object.wait(Object.java:485)
  2115. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2116. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2117. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2118. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2119. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2120. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2121. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2122. at java.lang.reflect.Method.invoke(Method.java:597)
  2123. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2124. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2125. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2126. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2127. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2128. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2129. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2130. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2131. at java.lang.reflect.Method.invoke(Method.java:597)
  2132. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2133. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2134.  
  2135. "IPC Server handler 11 on 60020" daemon prio=10 tid=0x00007f983c5bb000 nid=0x1b1e in Object.wait() [0x00007f9833bfa000]
  2136. java.lang.Thread.State: WAITING (on object monitor)
  2137. at java.lang.Object.wait(Native Method)
  2138. at java.lang.Object.wait(Object.java:485)
  2139. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2140. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2141. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2142. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2143. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2144. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2145. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2146. at java.lang.reflect.Method.invoke(Method.java:597)
  2147. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2148. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2149. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2150. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2151. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2152. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2153. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2154. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2155. at java.lang.reflect.Method.invoke(Method.java:597)
  2156. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2157. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2158.  
  2159. "IPC Server handler 10 on 60020" daemon prio=10 tid=0x00007f983c5b9000 nid=0x1b1d in Object.wait() [0x00007f9833cfb000]
  2160. java.lang.Thread.State: WAITING (on object monitor)
  2161. at java.lang.Object.wait(Native Method)
  2162. at java.lang.Object.wait(Object.java:485)
  2163. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  2164. - locked <0x0000000420ec2fa0> (a java.util.TreeSet)
  2165. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  2166. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  2167. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2168. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2169. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2170. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2171. at java.lang.reflect.Method.invoke(Method.java:597)
  2172. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2173. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2174.  
  2175. "IPC Server handler 9 on 60020" daemon prio=10 tid=0x00007f983c5b7000 nid=0x1b1c in Object.wait() [0x00007f9833dfc000]
  2176. java.lang.Thread.State: WAITING (on object monitor)
  2177. at java.lang.Object.wait(Native Method)
  2178. at java.lang.Object.wait(Object.java:485)
  2179. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2180. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2181. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2182. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2183. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2184. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2185. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2186. at java.lang.reflect.Method.invoke(Method.java:597)
  2187. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2188. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2189. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2190. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2191. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2192. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2193. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2194. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2195. at java.lang.reflect.Method.invoke(Method.java:597)
  2196. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2197. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2198.  
  2199. "IPC Server handler 8 on 60020" daemon prio=10 tid=0x00007f983c5b5000 nid=0x1b1b in Object.wait() [0x00007f9833efd000]
  2200. java.lang.Thread.State: WAITING (on object monitor)
  2201. at java.lang.Object.wait(Native Method)
  2202. at java.lang.Object.wait(Object.java:485)
  2203. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  2204. - locked <0x00000004254d7c78> (a java.util.TreeSet)
  2205. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  2206. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  2207. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2208. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2209. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2210. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2211. at java.lang.reflect.Method.invoke(Method.java:597)
  2212. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2213. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2214.  
  2215. "IPC Server handler 7 on 60020" daemon prio=10 tid=0x00007f983c598800 nid=0x1b1a in Object.wait() [0x00007f9833ffe000]
  2216. java.lang.Thread.State: WAITING (on object monitor)
  2217. at java.lang.Object.wait(Native Method)
  2218. at java.lang.Object.wait(Object.java:485)
  2219. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2220. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2221. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2222. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2223. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2224. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2225. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2226. at java.lang.reflect.Method.invoke(Method.java:597)
  2227. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2228. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2229. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2230. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2231. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2232. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2233. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2234. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2235. at java.lang.reflect.Method.invoke(Method.java:597)
  2236. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2237. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2238.  
  2239. "IPC Server handler 6 on 60020" daemon prio=10 tid=0x00007f983c596800 nid=0x1b19 in Object.wait() [0x00007f98381ed000]
  2240. java.lang.Thread.State: WAITING (on object monitor)
  2241. at java.lang.Object.wait(Native Method)
  2242. at java.lang.Object.wait(Object.java:485)
  2243. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2244. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2245. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2246. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2247. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2248. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2249. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2250. at java.lang.reflect.Method.invoke(Method.java:597)
  2251. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2252. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2253. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2254. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2255. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2256. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2257. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2258. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2259. at java.lang.reflect.Method.invoke(Method.java:597)
  2260. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2261. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2262.  
  2263. "IPC Server handler 5 on 60020" daemon prio=10 tid=0x00007f983c594800 nid=0x1b18 in Object.wait() [0x00007f98382ee000]
  2264. java.lang.Thread.State: WAITING (on object monitor)
  2265. at java.lang.Object.wait(Native Method)
  2266. at java.lang.Object.wait(Object.java:485)
  2267. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2268. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2269. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2270. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2271. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2272. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2273. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2274. at java.lang.reflect.Method.invoke(Method.java:597)
  2275. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2276. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2277. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2278. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2279. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2280. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2281. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2282. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2283. at java.lang.reflect.Method.invoke(Method.java:597)
  2284. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2285. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2286.  
  2287. "IPC Server handler 4 on 60020" daemon prio=10 tid=0x00007f983c592800 nid=0x1b17 in Object.wait() [0x00007f98383ef000]
  2288. java.lang.Thread.State: WAITING (on object monitor)
  2289. at java.lang.Object.wait(Native Method)
  2290. at java.lang.Object.wait(Object.java:485)
  2291. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2292. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2293. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2294. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2295. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2296. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2297. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2298. at java.lang.reflect.Method.invoke(Method.java:597)
  2299. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2300. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2301. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2302. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2303. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2304. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2305. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2306. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2307. at java.lang.reflect.Method.invoke(Method.java:597)
  2308. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2309. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2310.  
  2311. "IPC Server handler 3 on 60020" daemon prio=10 tid=0x00007f983c58c000 nid=0x1b16 in Object.wait() [0x00007f98384f0000]
  2312. java.lang.Thread.State: WAITING (on object monitor)
  2313. at java.lang.Object.wait(Native Method)
  2314. at java.lang.Object.wait(Object.java:485)
  2315. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2316. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2317. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2318. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2319. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2320. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2321. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2322. at java.lang.reflect.Method.invoke(Method.java:597)
  2323. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2324. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2325. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2326. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  2327. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  2328. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  2329. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  2330. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2331. at java.lang.reflect.Method.invoke(Method.java:597)
  2332. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2333. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2334.  
  2335. "IPC Server handler 2 on 60020" daemon prio=10 tid=0x00007f983c589000 nid=0x1b15 in Object.wait() [0x00007f98385f1000]
  2336. java.lang.Thread.State: WAITING (on object monitor)
  2337. at java.lang.Object.wait(Native Method)
  2338. at java.lang.Object.wait(Object.java:485)
  2339. at org.apache.hadoop.hbase.regionserver.HRegion.internalObtainRowLock(HRegion.java:2101)
  2340. - locked <0x000000041f192160> (a java.util.TreeSet)
  2341. at org.apache.hadoop.hbase.regionserver.HRegion.getLock(HRegion.java:2182)
  2342. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1433)
  2343. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2344. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2345. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2346. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2347. at java.lang.reflect.Method.invoke(Method.java:597)
  2348. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2349. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2350.  
  2351. "IPC Server handler 1 on 60020" daemon prio=10 tid=0x00007f983c588800 nid=0x1b14 in Object.wait() [0x00007f98386f2000]
  2352. java.lang.Thread.State: WAITING (on object monitor)
  2353. at java.lang.Object.wait(Native Method)
  2354. at java.lang.Object.wait(Object.java:485)
  2355. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2356. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2357. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2358. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2359. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2360. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2361. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2362. at java.lang.reflect.Method.invoke(Method.java:597)
  2363. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2364. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2365. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2366. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1255)
  2367. at org.apache.hadoop.hbase.regionserver.HRegion.delete(HRegion.java:1176)
  2368. at org.apache.hadoop.hbase.regionserver.HRegionServer.delete(HRegionServer.java:1905)
  2369. at sun.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
  2370. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2371. at java.lang.reflect.Method.invoke(Method.java:597)
  2372. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2373. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2374.  
  2375. "IPC Server handler 0 on 60020" daemon prio=10 tid=0x00007f983c57b000 nid=0x1b13 in Object.wait() [0x00007f98387f3000]
  2376. java.lang.Thread.State: WAITING (on object monitor)
  2377. at java.lang.Object.wait(Native Method)
  2378. at java.lang.Object.wait(Object.java:485)
  2379. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2380. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2381. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2382. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2383. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2384. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2385. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2386. at java.lang.reflect.Method.invoke(Method.java:597)
  2387. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2388. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2389. at org.apache.hadoop.hbase.regionserver.wal.HLog.append(HLog.java:937)
  2390. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchPut(HRegion.java:1477)
  2391. at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:1386)
  2392. at org.apache.hadoop.hbase.regionserver.HRegionServer.multi(HRegionServer.java:2549)
  2393. at sun.reflect.GeneratedMethodAccessor2.invoke(Unknown Source)
  2394. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2395. at java.lang.reflect.Method.invoke(Method.java:597)
  2396. at org.apache.hadoop.hbase.ipc.HBaseRPC$Server.call(HBaseRPC.java:570)
  2397. at org.apache.hadoop.hbase.ipc.HBaseServer$Handler.run(HBaseServer.java:1039)
  2398.  
  2399. "IPC Server listener on 60020" daemon prio=10 tid=0x00007f983c57a800 nid=0x1b12 waiting for monitor entry [0x00007f98388f4000]
  2400. java.lang.Thread.State: BLOCKED (on object monitor)
  2401. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.registerChannel(HBaseServer.java:347)
  2402. - waiting to lock <0x000000041d964510> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2403. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doAccept(HBaseServer.java:496)
  2404. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.run(HBaseServer.java:422)
  2405.  
  2406. "IPC Server Responder" daemon prio=10 tid=0x00007f983c579800 nid=0x1b11 runnable [0x00007f98389f5000]
  2407. java.lang.Thread.State: RUNNABLE
  2408. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2409. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  2410. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  2411. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  2412. - locked <0x000000041d9f4940> (a sun.nio.ch.Util$2)
  2413. - locked <0x000000041d9f4928> (a java.util.Collections$UnmodifiableSet)
  2414. - locked <0x000000041d9f3a98> (a sun.nio.ch.EPollSelectorImpl)
  2415. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  2416. at org.apache.hadoop.hbase.ipc.HBaseServer$Responder.run(HBaseServer.java:588)
  2417.  
  2418. "Timer-0" daemon prio=10 tid=0x00007f983c4ae000 nid=0x1b10 in Object.wait() [0x00007f9838af6000]
  2419. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2420. at java.lang.Object.wait(Native Method)
  2421. at java.util.TimerThread.mainLoop(Timer.java:509)
  2422. - locked <0x000000041eb23800> (a java.util.TaskQueue)
  2423. at java.util.TimerThread.run(Timer.java:462)
  2424.  
  2425. "1589789075@qtp-966979630-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030" prio=10 tid=0x00007f983c493000 nid=0x1b0d runnable [0x00007f9838bf7000]
  2426. java.lang.Thread.State: RUNNABLE
  2427. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2428. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  2429. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  2430. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  2431. - locked <0x000000041eab6310> (a sun.nio.ch.Util$2)
  2432. - locked <0x000000041eab6328> (a java.util.Collections$UnmodifiableSet)
  2433. - locked <0x000000041eaaec00> (a sun.nio.ch.EPollSelectorImpl)
  2434. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  2435. at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:498)
  2436. at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:192)
  2437. at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
  2438. at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
  2439. at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
  2440.  
  2441. "47973429@qtp-966979630-0" prio=10 tid=0x00007f983c49e800 nid=0x1b0c in Object.wait() [0x00007f9838cf8000]
  2442. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2443. at java.lang.Object.wait(Native Method)
  2444. at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
  2445. - locked <0x000000041df42208> (a org.mortbay.thread.QueuedThreadPool$PoolThread)
  2446.  
  2447. "regionserver60020.leaseChecker" daemon prio=10 tid=0x00007f983c48d800 nid=0x1b0b waiting on condition [0x00007f9838df9000]
  2448. java.lang.Thread.State: TIMED_WAITING (parking)
  2449. at sun.misc.Unsafe.park(Native Method)
  2450. - parking to wait for <0x000000041e4685e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2451. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  2452. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  2453. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
  2454. at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:82)
  2455.  
  2456. "regionserver60020.majorCompactionChecker" daemon prio=10 tid=0x00007f983c48b800 nid=0x1b0a in Object.wait() [0x00007f9838efa000]
  2457. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2458. at java.lang.Object.wait(Native Method)
  2459. at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:91)
  2460. - locked <0x000000041df5d800> (a java.lang.Object)
  2461. at org.apache.hadoop.hbase.Chore.run(Chore.java:74)
  2462.  
  2463. "regionserver60020.compactor" daemon prio=10 tid=0x00007f983c489800 nid=0x1b09 waiting on condition [0x00007f9838ffb000]
  2464. java.lang.Thread.State: TIMED_WAITING (parking)
  2465. at sun.misc.Unsafe.park(Native Method)
  2466. - parking to wait for <0x000000041d9c8d08> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2467. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  2468. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  2469. at java.util.concurrent.PriorityBlockingQueue.poll(PriorityBlockingQueue.java:245)
  2470. at org.apache.hadoop.hbase.regionserver.PriorityCompactionQueue.poll(PriorityCompactionQueue.java:248)
  2471. at org.apache.hadoop.hbase.regionserver.CompactSplitThread.run(CompactSplitThread.java:75)
  2472.  
  2473. "regionserver60020.cacheFlusher" daemon prio=10 tid=0x00007f983c488000 nid=0x1b08 waiting on condition [0x00007f98390fc000]
  2474. java.lang.Thread.State: TIMED_WAITING (parking)
  2475. at sun.misc.Unsafe.park(Native Method)
  2476. - parking to wait for <0x000000041da003d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2477. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  2478. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  2479. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:201)
  2480. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:39)
  2481. at org.apache.hadoop.hbase.regionserver.MemStoreFlusher.run(MemStoreFlusher.java:216)
  2482.  
  2483. "regionserver60020.logRoller" daemon prio=10 tid=0x00007f983c484800 nid=0x1b07 waiting for monitor entry [0x00007f98391fd000]
  2484. java.lang.Thread.State: BLOCKED (on object monitor)
  2485. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.writeChunk(DFSClient.java:3240)
  2486. - waiting to lock <0x000000059943a5d8> (a java.util.LinkedList)
  2487. - locked <0x00000005996d92a8> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream)
  2488. at org.apache.hadoop.fs.FSOutputSummer.writeChecksumChunk(FSOutputSummer.java:150)
  2489. at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:132)
  2490. - locked <0x00000005996d92a8> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream)
  2491. at org.apache.hadoop.fs.FSOutputSummer.flushBuffer(FSOutputSummer.java:121)
  2492. - locked <0x00000005996d92a8> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream)
  2493. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.closeInternal(DFSClient.java:3514)
  2494. - locked <0x00000005996d92a8> (a org.apache.hadoop.hdfs.DFSClient$DFSOutputStream)
  2495. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.close(DFSClient.java:3469)
  2496. at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:61)
  2497. at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:86)
  2498. at org.apache.hadoop.io.SequenceFile$Writer.close(SequenceFile.java:966)
  2499. - locked <0x000000059b284db0> (a org.apache.hadoop.io.SequenceFile$Writer)
  2500. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.close(SequenceFileLogWriter.java:138)
  2501. at org.apache.hadoop.hbase.regionserver.wal.HLog.cleanupCurrentWriter(HLog.java:719)
  2502. at org.apache.hadoop.hbase.regionserver.wal.HLog.rollWriter(HLog.java:504)
  2503. - locked <0x000000041ea8ba00> (a java.lang.Object)
  2504. at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:94)
  2505.  
  2506. "Timer thread for monitoring jvm" daemon prio=10 tid=0x00007f983c46b800 nid=0x1b06 in Object.wait() [0x00007f98392fe000]
  2507. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2508. at java.lang.Object.wait(Native Method)
  2509. at java.util.TimerThread.mainLoop(Timer.java:509)
  2510. - locked <0x000000041e468d18> (a java.util.TaskQueue)
  2511. at java.util.TimerThread.run(Timer.java:462)
  2512.  
  2513. "Timer thread for monitoring hbase" daemon prio=10 tid=0x00000000424c0000 nid=0x1b05 in Object.wait() [0x00007f98393ff000]
  2514. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2515. at java.lang.Object.wait(Native Method)
  2516. at java.util.TimerThread.mainLoop(Timer.java:509)
  2517. - locked <0x000000041da13e98> (a java.util.TaskQueue)
  2518. at java.util.TimerThread.run(Timer.java:462)
  2519.  
  2520. "regionserver60020.logSyncer" daemon prio=10 tid=0x00000000423d6800 nid=0x1b04 in Object.wait() [0x00007f9839500000]
  2521. java.lang.Thread.State: WAITING (on object monitor)
  2522. at java.lang.Object.wait(Native Method)
  2523. at java.lang.Object.wait(Object.java:485)
  2524. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.waitForAckedSeqno(DFSClient.java:3449)
  2525. - locked <0x000000059add8a28> (a java.util.LinkedList)
  2526. at org.apache.hadoop.hdfs.DFSClient$DFSOutputStream.sync(DFSClient.java:3366)
  2527. at org.apache.hadoop.fs.FSDataOutputStream.sync(FSDataOutputStream.java:97)
  2528. at org.apache.hadoop.io.SequenceFile$Writer.syncFs(SequenceFile.java:944)
  2529. at sun.reflect.GeneratedMethodAccessor6.invoke(Unknown Source)
  2530. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  2531. at java.lang.reflect.Method.invoke(Method.java:597)
  2532. at org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogWriter.sync(SequenceFileLogWriter.java:145)
  2533. at org.apache.hadoop.hbase.regionserver.wal.HLog.sync(HLog.java:986)
  2534. at org.apache.hadoop.hbase.regionserver.wal.HLog$LogSyncer.run(HLog.java:963)
  2535.  
  2536. "LeaseChecker" daemon prio=10 tid=0x0000000042355800 nid=0x1b03 waiting on condition [0x00007f9839601000]
  2537. java.lang.Thread.State: TIMED_WAITING (sleeping)
  2538. at java.lang.Thread.sleep(Native Method)
  2539. at org.apache.hadoop.hdfs.DFSClient$LeaseChecker.run(DFSClient.java:1151)
  2540. at java.lang.Thread.run(Thread.java:662)
  2541.  
  2542. "DestroyJavaVM" prio=10 tid=0x00007f983c45a800 nid=0x1abe waiting on condition [0x0000000000000000]
  2543. java.lang.Thread.State: RUNNABLE
  2544.  
  2545. "regionserver60020-EventThread" daemon prio=10 tid=0x000000004239f000 nid=0x1af9 waiting on condition [0x00007f9839c08000]
  2546. java.lang.Thread.State: WAITING (parking)
  2547. at sun.misc.Unsafe.park(Native Method)
  2548. - parking to wait for <0x000000041df5d818> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2549. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2550. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2551. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  2552. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
  2553.  
  2554. "regionserver60020-SendThread(prod00007.soniu.com:2181)" daemon prio=10 tid=0x000000004239e000 nid=0x1af8 runnable [0x00007f9839d09000]
  2555. java.lang.Thread.State: RUNNABLE
  2556. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2557. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  2558. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  2559. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  2560. - locked <0x000000041e468d48> (a sun.nio.ch.Util$2)
  2561. - locked <0x000000041e468d60> (a java.util.Collections$UnmodifiableSet)
  2562. - locked <0x000000041d9a6708> (a sun.nio.ch.EPollSelectorImpl)
  2563. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  2564. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1107)
  2565.  
  2566. "regionserver60020" prio=10 tid=0x00007f983c3bd000 nid=0x1af7 waiting on condition [0x00007f9839e0a000]
  2567. java.lang.Thread.State: TIMED_WAITING (parking)
  2568. at sun.misc.Unsafe.park(Native Method)
  2569. - parking to wait for <0x000000041df4ed30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2570. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:198)
  2571. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2025)
  2572. at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:424)
  2573. at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:624)
  2574. at java.lang.Thread.run(Thread.java:662)
  2575.  
  2576. "Timer thread for monitoring rpc" daemon prio=10 tid=0x00007f983c37b800 nid=0x1af3 in Object.wait() [0x00007f9839f0b000]
  2577. java.lang.Thread.State: TIMED_WAITING (on object monitor)
  2578. at java.lang.Object.wait(Native Method)
  2579. at java.util.TimerThread.mainLoop(Timer.java:509)
  2580. - locked <0x000000041d961cf8> (a java.util.TaskQueue)
  2581. at java.util.TimerThread.run(Timer.java:462)
  2582.  
  2583. "IPC Reader 9 on port 60020" prio=10 tid=0x00007f983c262800 nid=0x1af2 waiting on condition [0x00007f983a00c000]
  2584. java.lang.Thread.State: WAITING (parking)
  2585. at sun.misc.Unsafe.park(Native Method)
  2586. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2587. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2588. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2589. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2590. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2591. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2592. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2593. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2594. - locked <0x000000041d966e20> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2595. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2596. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2597. at java.lang.Thread.run(Thread.java:662)
  2598.  
  2599. "IPC Reader 8 on port 60020" prio=10 tid=0x00007f983c260800 nid=0x1af1 waiting on condition [0x00007f983a10d000]
  2600. java.lang.Thread.State: WAITING (parking)
  2601. at sun.misc.Unsafe.park(Native Method)
  2602. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2603. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2604. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2605. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2606. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2607. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2608. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2609. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2610. - locked <0x000000041d960870> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2611. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2612. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2613. at java.lang.Thread.run(Thread.java:662)
  2614.  
  2615. "IPC Reader 7 on port 60020" prio=10 tid=0x00007f983c25e800 nid=0x1af0 waiting on condition [0x00007f983a20e000]
  2616. java.lang.Thread.State: WAITING (parking)
  2617. at sun.misc.Unsafe.park(Native Method)
  2618. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2619. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2620. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2621. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2622. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2623. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2624. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2625. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2626. - locked <0x000000041d9643f0> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2627. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2628. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2629. at java.lang.Thread.run(Thread.java:662)
  2630.  
  2631. "IPC Reader 6 on port 60020" prio=10 tid=0x00007f983c25c800 nid=0x1aef waiting on condition [0x00007f983a30f000]
  2632. java.lang.Thread.State: WAITING (parking)
  2633. at sun.misc.Unsafe.park(Native Method)
  2634. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2635. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2636. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2637. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2638. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2639. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2640. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2641. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2642. - locked <0x000000041d964570> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2643. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2644. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2645. at java.lang.Thread.run(Thread.java:662)
  2646.  
  2647. "IPC Reader 5 on port 60020" prio=10 tid=0x00007f983c25a800 nid=0x1aee waiting on condition [0x00007f983a410000]
  2648. java.lang.Thread.State: WAITING (parking)
  2649. at sun.misc.Unsafe.park(Native Method)
  2650. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2651. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2652. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2653. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2654. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2655. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2656. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2657. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2658. - locked <0x000000041d96b350> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2659. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2660. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2661. at java.lang.Thread.run(Thread.java:662)
  2662.  
  2663. "IPC Reader 4 on port 60020" prio=10 tid=0x00007f983c258800 nid=0x1aed waiting on condition [0x00007f983a511000]
  2664. java.lang.Thread.State: WAITING (parking)
  2665. at sun.misc.Unsafe.park(Native Method)
  2666. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2667. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2668. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2669. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2670. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2671. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2672. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2673. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2674. - locked <0x000000041d964450> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2675. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2676. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2677. at java.lang.Thread.run(Thread.java:662)
  2678.  
  2679. "IPC Reader 3 on port 60020" prio=10 tid=0x00007f983c256800 nid=0x1aec waiting on condition [0x00007f983a612000]
  2680. java.lang.Thread.State: WAITING (parking)
  2681. at sun.misc.Unsafe.park(Native Method)
  2682. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2683. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2684. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2685. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2686. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2687. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2688. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2689. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2690. - locked <0x000000041d9645d0> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2691. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2692. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2693. at java.lang.Thread.run(Thread.java:662)
  2694.  
  2695. "IPC Reader 2 on port 60020" prio=10 tid=0x00007f983c255000 nid=0x1aeb waiting on condition [0x00007f983a713000]
  2696. java.lang.Thread.State: WAITING (parking)
  2697. at sun.misc.Unsafe.park(Native Method)
  2698. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2699. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2700. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2701. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2702. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2703. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2704. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2705. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2706. - locked <0x000000041d9644b0> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2707. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2708. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2709. at java.lang.Thread.run(Thread.java:662)
  2710.  
  2711. "IPC Reader 1 on port 60020" prio=10 tid=0x00007f983c253000 nid=0x1aea waiting on condition [0x00007f983a814000]
  2712. java.lang.Thread.State: WAITING (parking)
  2713. at sun.misc.Unsafe.park(Native Method)
  2714. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2715. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2716. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2717. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2718. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2719. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2720. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2721. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2722. - locked <0x000000041d964630> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2723. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2724. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2725. at java.lang.Thread.run(Thread.java:662)
  2726.  
  2727. "IPC Reader 0 on port 60020" prio=10 tid=0x00007f983c1aa800 nid=0x1ae9 waiting on condition [0x00007f983a915000]
  2728. java.lang.Thread.State: WAITING (parking)
  2729. at sun.misc.Unsafe.park(Native Method)
  2730. - parking to wait for <0x000000041d9f51f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2731. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2732. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2733. at java.util.concurrent.LinkedBlockingQueue.put(LinkedBlockingQueue.java:306)
  2734. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.processData(HBaseServer.java:985)
  2735. at org.apache.hadoop.hbase.ipc.HBaseServer$Connection.readAndProcess(HBaseServer.java:946)
  2736. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener.doRead(HBaseServer.java:522)
  2737. at org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader.run(HBaseServer.java:316)
  2738. - locked <0x000000041d964510> (a org.apache.hadoop.hbase.ipc.HBaseServer$Listener$Reader)
  2739. at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
  2740. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
  2741. at java.lang.Thread.run(Thread.java:662)
  2742.  
  2743. "main-EventThread" daemon prio=10 tid=0x00007f983c1a9800 nid=0x1ae7 waiting on condition [0x00007f983ae32000]
  2744. java.lang.Thread.State: WAITING (parking)
  2745. at sun.misc.Unsafe.park(Native Method)
  2746. - parking to wait for <0x000000041d964660> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  2747. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
  2748. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
  2749. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
  2750. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
  2751.  
  2752. "main-SendThread(prod00006.soniu.com:2181)" daemon prio=10 tid=0x00007f983c1be800 nid=0x1ae6 runnable [0x00007f983af33000]
  2753. java.lang.Thread.State: RUNNABLE
  2754. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  2755. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:210)
  2756. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:65)
  2757. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:69)
  2758. - locked <0x000000041d966470> (a sun.nio.ch.Util$2)
  2759. - locked <0x000000041d966488> (a java.util.Collections$UnmodifiableSet)
  2760. - locked <0x000000041d963eb0> (a sun.nio.ch.EPollSelectorImpl)
  2761. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:80)
  2762. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1107)
  2763.  
  2764. "Low Memory Detector" daemon prio=10 tid=0x00007f983c006800 nid=0x1ad2 runnable [0x0000000000000000]
  2765. java.lang.Thread.State: RUNNABLE
  2766.  
  2767. "CompilerThread1" daemon prio=10 tid=0x00007f983c003800 nid=0x1ad1 waiting on condition [0x0000000000000000]
  2768. java.lang.Thread.State: RUNNABLE
  2769.  
  2770. "CompilerThread0" daemon prio=10 tid=0x00007f983c001000 nid=0x1ad0 waiting on condition [0x0000000000000000]
  2771. java.lang.Thread.State: RUNNABLE
  2772.  
  2773. "Signal Dispatcher" daemon prio=10 tid=0x0000000041cd0000 nid=0x1acf runnable [0x0000000000000000]
  2774. java.lang.Thread.State: RUNNABLE
  2775.  
  2776. "Surrogate Locker Thread (CMS)" daemon prio=10 tid=0x0000000041cce000 nid=0x1ace waiting on condition [0x0000000000000000]
  2777. java.lang.Thread.State: RUNNABLE
  2778.  
  2779. "Finalizer" daemon prio=10 tid=0x0000000041caf800 nid=0x1acc in Object.wait() [0x00007f983bedb000]
  2780. java.lang.Thread.State: WAITING (on object monitor)
  2781. at java.lang.Object.wait(Native Method)
  2782. at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
  2783. - locked <0x000000041d9646a8> (a java.lang.ref.ReferenceQueue$Lock)
  2784. at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
  2785. at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
  2786.  
  2787. "Reference Handler" daemon prio=10 tid=0x0000000041ca8000 nid=0x1acb in Object.wait() [0x00007f983bfdc000]
  2788. java.lang.Thread.State: WAITING (on object monitor)
  2789. at java.lang.Object.wait(Native Method)
  2790. at java.lang.Object.wait(Object.java:485)
  2791. at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
  2792. - locked <0x000000041d960028> (a java.lang.ref.Reference$Lock)
  2793.  
  2794. "VM Thread" prio=10 tid=0x0000000041ca3800 nid=0x1aca runnable
  2795.  
  2796. "Gang worker#0 (Parallel GC Threads)" prio=10 tid=0x0000000041ac5800 nid=0x1abf runnable
  2797.  
  2798. "Gang worker#1 (Parallel GC Threads)" prio=10 tid=0x0000000041ac7800 nid=0x1ac0 runnable
  2799.  
  2800. "Gang worker#2 (Parallel GC Threads)" prio=10 tid=0x0000000041ac9800 nid=0x1ac1 runnable
  2801.  
  2802. "Gang worker#3 (Parallel GC Threads)" prio=10 tid=0x0000000041acb800 nid=0x1ac2 runnable
  2803.  
  2804. "Gang worker#4 (Parallel GC Threads)" prio=10 tid=0x0000000041acd000 nid=0x1ac3 runnable
  2805.  
  2806. "Gang worker#5 (Parallel GC Threads)" prio=10 tid=0x0000000041acf000 nid=0x1ac4 runnable
  2807.  
  2808. "Gang worker#6 (Parallel GC Threads)" prio=10 tid=0x0000000041ad1000 nid=0x1ac5 runnable
  2809.  
  2810. "Gang worker#7 (Parallel GC Threads)" prio=10 tid=0x0000000041ad2800 nid=0x1ac6 runnable
  2811.  
  2812. "Concurrent Mark-Sweep GC Thread" prio=10 tid=0x0000000041bc6800 nid=0x1ac9 runnable
  2813. "Gang worker#0 (Parallel CMS Threads)" prio=10 tid=0x0000000041bc2800 nid=0x1ac7 runnable
  2814.  
  2815. "Gang worker#1 (Parallel CMS Threads)" prio=10 tid=0x0000000041bc4000 nid=0x1ac8 runnable
  2816.  
  2817. "VM Periodic Task Thread" prio=10 tid=0x00007f9834002800 nid=0x1ad3 waiting on condition
  2818.  
  2819. JNI global references: 1362
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!