API tools faq
paste
Advertisement
Guest User

hot region server stack trace

a guest
Dec 2nd, 2016
177
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 304.38 KB | None | 0 0
raw download clone embed print report
  1. 2016-12-02 14:14:16
  2. Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.66-b17 mixed mode):
  3.  
  4. "RMI TCP Connection(118)-10.172.226.11" - Thread t@3408386
  5. java.lang.Thread.State: RUNNABLE
  6. at java.net.SocketInputStream.socketRead0(Native Method)
  7. at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
  8. at java.net.SocketInputStream.read(SocketInputStream.java:170)
  9. at java.net.SocketInputStream.read(SocketInputStream.java:141)
  10. at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
  11. at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
  12. - locked <29864006> (a java.io.BufferedInputStream)
  13. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  14. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
  15. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  16. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  17. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  18. at java.security.AccessController.doPrivileged(Native Method)
  19. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  20. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  21. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  22. at java.lang.Thread.run(Thread.java:745)
  23.  
  24. Locked ownable synchronizers:
  25. - locked <294f414d> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  26.  
  27. "RMI TCP Connection(117)-10.172.226.11" - Thread t@3408385
  28. java.lang.Thread.State: TIMED_WAITING
  29. at java.lang.Object.wait(Native Method)
  30. - waiting on <18400d31> (a com.sun.jmx.remote.internal.ArrayNotificationBuffer)
  31. at com.sun.jmx.remote.internal.ArrayNotificationBuffer.fetchNotifications(ArrayNotificationBuffer.java:449)
  32. at com.sun.jmx.remote.internal.ArrayNotificationBuffer$ShareBuffer.fetchNotifications(ArrayNotificationBuffer.java:227)
  33. at com.sun.jmx.remote.internal.ServerNotifForwarder.fetchNotifs(ServerNotifForwarder.java:274)
  34. at javax.management.remote.rmi.RMIConnectionImpl$4.run(RMIConnectionImpl.java:1273)
  35. at javax.management.remote.rmi.RMIConnectionImpl$4.run(RMIConnectionImpl.java:1271)
  36. at javax.management.remote.rmi.RMIConnectionImpl.fetchNotifications(RMIConnectionImpl.java:1277)
  37. at sun.reflect.GeneratedMethodAccessor247.invoke(Unknown Source)
  38. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  39. at java.lang.reflect.Method.invoke(Method.java:497)
  40. at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
  41. at sun.rmi.transport.Transport$1.run(Transport.java:200)
  42. at sun.rmi.transport.Transport$1.run(Transport.java:197)
  43. at java.security.AccessController.doPrivileged(Native Method)
  44. at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
  45. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
  46. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  47. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  48. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  49. at java.security.AccessController.doPrivileged(Native Method)
  50. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  51. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  52. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  53. at java.lang.Thread.run(Thread.java:745)
  54.  
  55. Locked ownable synchronizers:
  56. - locked <30de6668> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  57.  
  58. "ResponseProcessor for block BP-1025249305-10.99.183.228-1458740156596:blk_1382454725_308714360" - Thread t@3408384
  59. java.lang.Thread.State: RUNNABLE
  60. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  61. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  62. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  63. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  64. - locked <1eb72723> (a sun.nio.ch.Util$2)
  65. - locked <7e45ad59> (a java.util.Collections$UnmodifiableSet)
  66. - locked <3121c3b9> (a sun.nio.ch.EPollSelectorImpl)
  67. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  68. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
  69. at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  70. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
  71. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
  72. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118)
  73. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  74. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  75. at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2239)
  76. at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:235)
  77. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:971)
  78.  
  79. Locked ownable synchronizers:
  80. - None
  81.  
  82. "RMI TCP Connection(119)-10.172.226.11" - Thread t@3408383
  83. java.lang.Thread.State: RUNNABLE
  84. at sun.management.ThreadImpl.dumpThreads0(Native Method)
  85. at sun.management.ThreadImpl.dumpAllThreads(ThreadImpl.java:446)
  86. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  87. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  88. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  89. at java.lang.reflect.Method.invoke(Method.java:497)
  90. at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:71)
  91. at sun.reflect.GeneratedMethodAccessor91.invoke(Unknown Source)
  92. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  93. at java.lang.reflect.Method.invoke(Method.java:497)
  94. at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:275)
  95. at com.sun.jmx.mbeanserver.ConvertingMethod.invokeWithOpenReturn(ConvertingMethod.java:193)
  96. at com.sun.jmx.mbeanserver.ConvertingMethod.invokeWithOpenReturn(ConvertingMethod.java:175)
  97. at com.sun.jmx.mbeanserver.MXBeanIntrospector.invokeM2(MXBeanIntrospector.java:117)
  98. at com.sun.jmx.mbeanserver.MXBeanIntrospector.invokeM2(MXBeanIntrospector.java:54)
  99. at com.sun.jmx.mbeanserver.MBeanIntrospector.invokeM(MBeanIntrospector.java:237)
  100. at com.sun.jmx.mbeanserver.PerInterface.invoke(PerInterface.java:138)
  101. at com.sun.jmx.mbeanserver.MBeanSupport.invoke(MBeanSupport.java:252)
  102. at javax.management.StandardMBean.invoke(StandardMBean.java:405)
  103. at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.invoke(DefaultMBeanServerInterceptor.java:819)
  104. at com.sun.jmx.mbeanserver.JmxMBeanServer.invoke(JmxMBeanServer.java:801)
  105. at javax.management.remote.rmi.RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1471)
  106. at javax.management.remote.rmi.RMIConnectionImpl.access$300(RMIConnectionImpl.java:76)
  107. at javax.management.remote.rmi.RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1312)
  108. at javax.management.remote.rmi.RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1404)
  109. at javax.management.remote.rmi.RMIConnectionImpl.invoke(RMIConnectionImpl.java:832)
  110. at sun.reflect.GeneratedMethodAccessor241.invoke(Unknown Source)
  111. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  112. at java.lang.reflect.Method.invoke(Method.java:497)
  113. at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
  114. at sun.rmi.transport.Transport$1.run(Transport.java:200)
  115. at sun.rmi.transport.Transport$1.run(Transport.java:197)
  116. at java.security.AccessController.doPrivileged(Native Method)
  117. at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
  118. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
  119. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  120. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  121. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  122. at java.security.AccessController.doPrivileged(Native Method)
  123. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  124. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  125. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  126. at java.lang.Thread.run(Thread.java:745)
  127.  
  128. Locked ownable synchronizers:
  129. - locked <1b43bb10> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  130.  
  131. "RMI TCP Connection(115)-10.172.226.11" - Thread t@3408380
  132. java.lang.Thread.State: RUNNABLE
  133. at java.net.SocketInputStream.socketRead0(Native Method)
  134. at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
  135. at java.net.SocketInputStream.read(SocketInputStream.java:170)
  136. at java.net.SocketInputStream.read(SocketInputStream.java:141)
  137. at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
  138. at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
  139. - locked <2a3c33eb> (a java.io.BufferedInputStream)
  140. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  141. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
  142. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  143. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  144. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  145. at java.security.AccessController.doPrivileged(Native Method)
  146. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  147. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  148. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  149. at java.lang.Thread.run(Thread.java:745)
  150.  
  151. Locked ownable synchronizers:
  152. - locked <38e1ebe2> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  153.  
  154. "ResponseProcessor for block BP-1025249305-10.99.183.228-1458740156596:blk_1382454486_308714121" - Thread t@3408379
  155. java.lang.Thread.State: RUNNABLE
  156. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  157. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  158. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  159. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  160. - locked <d9597b0> (a sun.nio.ch.Util$2)
  161. - locked <5e69cc3e> (a java.util.Collections$UnmodifiableSet)
  162. - locked <616af041> (a sun.nio.ch.EPollSelectorImpl)
  163. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  164. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
  165. at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  166. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
  167. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
  168. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118)
  169. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  170. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  171. at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2239)
  172. at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:235)
  173. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:971)
  174.  
  175. Locked ownable synchronizers:
  176. - None
  177.  
  178. "DataStreamer for file /hbase/WALs/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net,60020,1480229275047/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net%2C60020%2C1480229275047.null0.1480706014688 block BP-1025249305-10.99.183.228-1458740156596:blk_1382454486_308714121" - Thread t@3408378
  179. java.lang.Thread.State: TIMED_WAITING
  180. at java.lang.Object.wait(Native Method)
  181. - waiting on <67ba892d> (a java.util.LinkedList)
  182. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:638)
  183.  
  184. Locked ownable synchronizers:
  185. - None
  186.  
  187. "IPC Client (1414013111) connection to ip-10-99-154-54.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.54:60020 from hbase" - Thread t@3408377
  188. java.lang.Thread.State: TIMED_WAITING
  189. at java.lang.Object.wait(Native Method)
  190. - waiting on <f4fe41c> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  191. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  192. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  193.  
  194. Locked ownable synchronizers:
  195. - None
  196.  
  197. "JMX server connection timeout 3408374" - Thread t@3408374
  198. java.lang.Thread.State: TIMED_WAITING
  199. at java.lang.Object.wait(Native Method)
  200. - waiting on <50912f7c> (a [I)
  201. at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:168)
  202. at java.lang.Thread.run(Thread.java:745)
  203.  
  204. Locked ownable synchronizers:
  205. - None
  206.  
  207. "RMI TCP Connection(114)-10.172.226.11" - Thread t@3408373
  208. java.lang.Thread.State: RUNNABLE
  209. at java.net.SocketInputStream.socketRead0(Native Method)
  210. at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
  211. at java.net.SocketInputStream.read(SocketInputStream.java:170)
  212. at java.net.SocketInputStream.read(SocketInputStream.java:141)
  213. at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
  214. at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
  215. - locked <2a8234ec> (a java.io.BufferedInputStream)
  216. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  217. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
  218. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  219. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  220. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  221. at java.security.AccessController.doPrivileged(Native Method)
  222. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  223. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  224. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  225. at java.lang.Thread.run(Thread.java:745)
  226.  
  227. Locked ownable synchronizers:
  228. - locked <1d809117> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  229.  
  230. "RMI TCP Connection(120)-10.172.226.11" - Thread t@3408372
  231. java.lang.Thread.State: RUNNABLE
  232. at java.net.SocketInputStream.socketRead0(Native Method)
  233. at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
  234. at java.net.SocketInputStream.read(SocketInputStream.java:170)
  235. at java.net.SocketInputStream.read(SocketInputStream.java:141)
  236. at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
  237. at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
  238. - locked <b8b0e1b> (a java.io.BufferedInputStream)
  239. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  240. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
  241. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  242. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  243. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  244. at java.security.AccessController.doPrivileged(Native Method)
  245. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  246. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  247. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  248. at java.lang.Thread.run(Thread.java:745)
  249.  
  250. Locked ownable synchronizers:
  251. - locked <169f3d25> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  252.  
  253. "IPC Client (1414013111) connection to ip-10-99-154-112.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.112:60020 from hbase" - Thread t@3408309
  254. java.lang.Thread.State: TIMED_WAITING
  255. at java.lang.Object.wait(Native Method)
  256. - waiting on <5396e8d1> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  257. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  258. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  259.  
  260. Locked ownable synchronizers:
  261. - None
  262.  
  263. "IPC Client (1414013111) connection to ip-10-99-154-157.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.157:60020 from hbase" - Thread t@3408292
  264. java.lang.Thread.State: TIMED_WAITING
  265. at java.lang.Object.wait(Native Method)
  266. - waiting on <56c0d680> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  267. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  268. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  269.  
  270. Locked ownable synchronizers:
  271. - None
  272.  
  273. "IPC Client (1414013111) connection to ip-10-99-154-116.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.116:60020 from hbase" - Thread t@3408235
  274. java.lang.Thread.State: TIMED_WAITING
  275. at java.lang.Object.wait(Native Method)
  276. - waiting on <715e72ca> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  277. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  278. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  279.  
  280. Locked ownable synchronizers:
  281. - None
  282.  
  283. "DataStreamer for file /hbase/data/default/pgs-device/8c1ac0ed31be8879d635330257b683a0/.tmp/2abe3b42821440019eda2217d3f9065e block BP-1025249305-10.99.183.228-1458740156596:blk_1382454725_308714360" - Thread t@3408193
  284. java.lang.Thread.State: TIMED_WAITING
  285. at java.lang.Object.wait(Native Method)
  286. - waiting on <23c44c1c> (a java.util.LinkedList)
  287. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:638)
  288.  
  289. Locked ownable synchronizers:
  290. - None
  291.  
  292. "IPC Client (1414013111) connection to ip-10-99-153-50.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.153.50:60020 from hbase" - Thread t@3408192
  293. java.lang.Thread.State: TIMED_WAITING
  294. at java.lang.Object.wait(Native Method)
  295. - waiting on <29adcb5a> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  296. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  297. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  298.  
  299. Locked ownable synchronizers:
  300. - None
  301.  
  302. "IPC Client (1414013111) connection to ip-10-99-154-158.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.158:60020 from hbase" - Thread t@3408173
  303. java.lang.Thread.State: TIMED_WAITING
  304. at java.lang.Object.wait(Native Method)
  305. - waiting on <59687ef8> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  306. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  307. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  308.  
  309. Locked ownable synchronizers:
  310. - None
  311.  
  312. "RMI TCP Connection(112)-10.172.98.36" - Thread t@3408037
  313. java.lang.Thread.State: RUNNABLE
  314. at java.net.SocketInputStream.socketRead0(Native Method)
  315. at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
  316. at java.net.SocketInputStream.read(SocketInputStream.java:170)
  317. at java.net.SocketInputStream.read(SocketInputStream.java:141)
  318. at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
  319. at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
  320. - locked <1b933b68> (a java.io.BufferedInputStream)
  321. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  322. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
  323. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  324. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  325. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  326. at java.security.AccessController.doPrivileged(Native Method)
  327. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  328. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  329. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  330. at java.lang.Thread.run(Thread.java:745)
  331.  
  332. Locked ownable synchronizers:
  333. - locked <26f1b8e8> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  334.  
  335. "IPC Client (1414013111) connection to ip-10-99-154-55.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.55:60020 from hbase" - Thread t@3406593
  336. java.lang.Thread.State: TIMED_WAITING
  337. at java.lang.Object.wait(Native Method)
  338. - waiting on <b0cbda6> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  339. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  340. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  341.  
  342. Locked ownable synchronizers:
  343. - None
  344.  
  345. "IPC Client (1414013111) connection to ip-10-99-154-138.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.138:60020 from hbase" - Thread t@3406554
  346. java.lang.Thread.State: TIMED_WAITING
  347. at java.lang.Object.wait(Native Method)
  348. - waiting on <5f52dc1c> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  349. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  350. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  351.  
  352. Locked ownable synchronizers:
  353. - None
  354.  
  355. "IPC Client (1414013111) connection to ip-10-99-153-48.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.153.48:60020 from hbase" - Thread t@3406470
  356. java.lang.Thread.State: TIMED_WAITING
  357. at java.lang.Object.wait(Native Method)
  358. - waiting on <79ad8a34> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  359. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  360. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  361.  
  362. Locked ownable synchronizers:
  363. - None
  364.  
  365. "IPC Client (1414013111) connection to ip-10-99-154-60.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.60:60020 from hbase" - Thread t@3406430
  366. java.lang.Thread.State: TIMED_WAITING
  367. at java.lang.Object.wait(Native Method)
  368. - waiting on <6b8e3c14> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  369. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  370. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  371.  
  372. Locked ownable synchronizers:
  373. - None
  374.  
  375. "947501014@qtp-1164622694-99" - Thread t@3406418
  376. java.lang.Thread.State: TIMED_WAITING
  377. at java.lang.Object.wait(Native Method)
  378. - waiting on <3879b7d6> (a org.mortbay.thread.QueuedThreadPool$PoolThread)
  379. at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:626)
  380.  
  381. Locked ownable synchronizers:
  382. - None
  383.  
  384. "IPC Client (1414013111) connection to ip-10-99-154-107.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.107:60020 from hbase" - Thread t@3405837
  385. java.lang.Thread.State: TIMED_WAITING
  386. at java.lang.Object.wait(Native Method)
  387. - waiting on <2958f50> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  388. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  389. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  390.  
  391. Locked ownable synchronizers:
  392. - None
  393.  
  394. "IPC Client (1414013111) connection to ip-10-99-154-161.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.161:60020 from hbase" - Thread t@3405808
  395. java.lang.Thread.State: TIMED_WAITING
  396. at java.lang.Object.wait(Native Method)
  397. - waiting on <47a9ab5> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  398. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  399. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  400.  
  401. Locked ownable synchronizers:
  402. - None
  403.  
  404. "IPC Client (1414013111) connection to ip-10-99-153-49.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.153.49:60020 from hbase" - Thread t@3404956
  405. java.lang.Thread.State: TIMED_WAITING
  406. at java.lang.Object.wait(Native Method)
  407. - waiting on <307e0277> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  408. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  409. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  410.  
  411. Locked ownable synchronizers:
  412. - None
  413.  
  414. "IPC Client (1414013111) connection to ip-10-99-154-122.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.122:60020 from hbase" - Thread t@3404565
  415. java.lang.Thread.State: TIMED_WAITING
  416. at java.lang.Object.wait(Native Method)
  417. - waiting on <69b20125> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  418. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  419. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  420.  
  421. Locked ownable synchronizers:
  422. - None
  423.  
  424. "IPC Client (1414013111) connection to ip-10-99-152-9.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.152.9:60020 from hbase" - Thread t@3404454
  425. java.lang.Thread.State: TIMED_WAITING
  426. at java.lang.Object.wait(Native Method)
  427. - waiting on <68e464c9> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  428. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  429. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  430.  
  431. Locked ownable synchronizers:
  432. - None
  433.  
  434. "IPC Client (1414013111) connection to ip-10-99-152-4.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.152.4:60020 from hbase" - Thread t@3404436
  435. java.lang.Thread.State: TIMED_WAITING
  436. at java.lang.Object.wait(Native Method)
  437. - waiting on <19f6cef6> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  438. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  439. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  440.  
  441. Locked ownable synchronizers:
  442. - None
  443.  
  444. "IPC Client (1414013111) connection to ip-10-99-154-56.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.56:60020 from hbase" - Thread t@3403219
  445. java.lang.Thread.State: TIMED_WAITING
  446. at java.lang.Object.wait(Native Method)
  447. - waiting on <38bd2519> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  448. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  449. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  450.  
  451. Locked ownable synchronizers:
  452. - None
  453.  
  454. "IPC Client (1414013111) connection to ip-10-99-154-59.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.59:60020 from hbase" - Thread t@3403042
  455. java.lang.Thread.State: TIMED_WAITING
  456. at java.lang.Object.wait(Native Method)
  457. - waiting on <5177fb07> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  458. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  459. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  460.  
  461. Locked ownable synchronizers:
  462. - None
  463.  
  464. "IPC Client (1414013111) connection to ip-10-99-154-117.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.117:60020 from hbase" - Thread t@3402730
  465. java.lang.Thread.State: TIMED_WAITING
  466. at java.lang.Object.wait(Native Method)
  467. - waiting on <1fd55101> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  468. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  469. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  470.  
  471. Locked ownable synchronizers:
  472. - None
  473.  
  474. "IPC Client (1414013111) connection to ip-10-99-154-144.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.144:60020 from hbase" - Thread t@3402641
  475. java.lang.Thread.State: TIMED_WAITING
  476. at java.lang.Object.wait(Native Method)
  477. - waiting on <39bd1663> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  478. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  479. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  480.  
  481. Locked ownable synchronizers:
  482. - None
  483.  
  484. "RMI TCP Connection(88)-10.172.98.36" - Thread t@3401469
  485. java.lang.Thread.State: TIMED_WAITING
  486. at java.lang.Object.wait(Native Method)
  487. - waiting on <18400d31> (a com.sun.jmx.remote.internal.ArrayNotificationBuffer)
  488. at com.sun.jmx.remote.internal.ArrayNotificationBuffer.fetchNotifications(ArrayNotificationBuffer.java:449)
  489. at com.sun.jmx.remote.internal.ArrayNotificationBuffer$ShareBuffer.fetchNotifications(ArrayNotificationBuffer.java:227)
  490. at com.sun.jmx.remote.internal.ServerNotifForwarder.fetchNotifs(ServerNotifForwarder.java:274)
  491. at javax.management.remote.rmi.RMIConnectionImpl$4.run(RMIConnectionImpl.java:1273)
  492. at javax.management.remote.rmi.RMIConnectionImpl$4.run(RMIConnectionImpl.java:1271)
  493. at javax.management.remote.rmi.RMIConnectionImpl.fetchNotifications(RMIConnectionImpl.java:1277)
  494. at sun.reflect.GeneratedMethodAccessor247.invoke(Unknown Source)
  495. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  496. at java.lang.reflect.Method.invoke(Method.java:497)
  497. at sun.rmi.server.UnicastServerRef.dispatch(UnicastServerRef.java:323)
  498. at sun.rmi.transport.Transport$1.run(Transport.java:200)
  499. at sun.rmi.transport.Transport$1.run(Transport.java:197)
  500. at java.security.AccessController.doPrivileged(Native Method)
  501. at sun.rmi.transport.Transport.serviceCall(Transport.java:196)
  502. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:568)
  503. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  504. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  505. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  506. at java.security.AccessController.doPrivileged(Native Method)
  507. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  508. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  509. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  510. at java.lang.Thread.run(Thread.java:745)
  511.  
  512. Locked ownable synchronizers:
  513. - locked <5fa90361> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  514.  
  515. "IPC Client (1414013111) connection to ip-10-99-154-149.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.149:60020 from hbase" - Thread t@3401436
  516. java.lang.Thread.State: TIMED_WAITING
  517. at java.lang.Object.wait(Native Method)
  518. - waiting on <48875595> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  519. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  520. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  521.  
  522. Locked ownable synchronizers:
  523. - None
  524.  
  525. "IPC Client (1414013111) connection to ip-10-99-154-145.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.145:60020 from hbase" - Thread t@3401404
  526. java.lang.Thread.State: TIMED_WAITING
  527. at java.lang.Object.wait(Native Method)
  528. - waiting on <11024be2> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  529. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  530. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  531.  
  532. Locked ownable synchronizers:
  533. - None
  534.  
  535. "JMX server connection timeout 3401339" - Thread t@3401339
  536. java.lang.Thread.State: TIMED_WAITING
  537. at java.lang.Object.wait(Native Method)
  538. - waiting on <5fafe9a5> (a [I)
  539. at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:168)
  540. at java.lang.Thread.run(Thread.java:745)
  541.  
  542. Locked ownable synchronizers:
  543. - None
  544.  
  545. "RMI TCP Connection(109)-10.99.181.146" - Thread t@3401144
  546. java.lang.Thread.State: RUNNABLE
  547. at java.net.SocketInputStream.socketRead0(Native Method)
  548. at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
  549. at java.net.SocketInputStream.read(SocketInputStream.java:170)
  550. at java.net.SocketInputStream.read(SocketInputStream.java:141)
  551. at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
  552. at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
  553. - locked <2d4d28a3> (a java.io.BufferedInputStream)
  554. at java.io.FilterInputStream.read(FilterInputStream.java:83)
  555. at sun.rmi.transport.tcp.TCPTransport.handleMessages(TCPTransport.java:550)
  556. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run0(TCPTransport.java:826)
  557. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.lambda$run$95(TCPTransport.java:683)
  558. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler$$Lambda$9/494225322.run(Unknown Source)
  559. at java.security.AccessController.doPrivileged(Native Method)
  560. at sun.rmi.transport.tcp.TCPTransport$ConnectionHandler.run(TCPTransport.java:682)
  561. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  562. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  563. at java.lang.Thread.run(Thread.java:745)
  564.  
  565. Locked ownable synchronizers:
  566. - locked <27711851> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  567.  
  568. "IPC Client (1414013111) connection to ip-10-99-154-120.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.120:60020 from hbase" - Thread t@3400463
  569. java.lang.Thread.State: TIMED_WAITING
  570. at java.lang.Object.wait(Native Method)
  571. - waiting on <1153dad> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  572. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  573. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  574.  
  575. Locked ownable synchronizers:
  576. - None
  577.  
  578. "IPC Client (1414013111) connection to ip-10-99-154-129.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.129:60020 from hbase" - Thread t@3400460
  579. java.lang.Thread.State: TIMED_WAITING
  580. at java.lang.Object.wait(Native Method)
  581. - waiting on <72453004> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  582. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  583. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  584.  
  585. Locked ownable synchronizers:
  586. - None
  587.  
  588. "IPC Client (1414013111) connection to ip-10-99-154-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.146:60020 from hbase" - Thread t@3400405
  589. java.lang.Thread.State: TIMED_WAITING
  590. at java.lang.Object.wait(Native Method)
  591. - waiting on <74852c9> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  592. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  593. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  594.  
  595. Locked ownable synchronizers:
  596. - None
  597.  
  598. "IPC Parameter Sending Thread #826" - Thread t@3393805
  599. java.lang.Thread.State: TIMED_WAITING
  600. at sun.misc.Unsafe.park(Native Method)
  601. - parking to wait for <482eddd9> (a java.util.concurrent.SynchronousQueue$TransferStack)
  602. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  603. at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
  604. at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
  605. at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
  606. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
  607. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  608. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  609. at java.lang.Thread.run(Thread.java:745)
  610.  
  611. Locked ownable synchronizers:
  612. - None
  613.  
  614. "IPC Client (1414013111) connection to ip-10-99-154-127.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.127:60020 from hbase" - Thread t@3371652
  615. java.lang.Thread.State: TIMED_WAITING
  616. at java.lang.Object.wait(Native Method)
  617. - waiting on <25074ea3> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  618. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  619. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  620.  
  621. Locked ownable synchronizers:
  622. - None
  623.  
  624. "IPC Client (1414013111) connection to ip-10-99-154-150.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.150:60020 from hbase" - Thread t@3352700
  625. java.lang.Thread.State: TIMED_WAITING
  626. at java.lang.Object.wait(Native Method)
  627. - waiting on <73cdb809> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  628. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  629. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  630.  
  631. Locked ownable synchronizers:
  632. - None
  633.  
  634. "IPC Client (1414013111) connection to ip-10-99-154-121.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.121:60020 from hbase" - Thread t@3325168
  635. java.lang.Thread.State: TIMED_WAITING
  636. at java.lang.Object.wait(Native Method)
  637. - waiting on <61953844> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  638. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  639. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  640.  
  641. Locked ownable synchronizers:
  642. - None
  643.  
  644. "IPC Client (1414013111) connection to ip-10-99-154-131.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.131:60020 from hbase" - Thread t@3117418
  645. java.lang.Thread.State: TIMED_WAITING
  646. at java.lang.Object.wait(Native Method)
  647. - waiting on <7393c6a9> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  648. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  649. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  650.  
  651. Locked ownable synchronizers:
  652. - None
  653.  
  654. "IPC Client (1414013111) connection to ip-10-99-183-228.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.183.228:8020 from hbase" - Thread t@3087997
  655. java.lang.Thread.State: TIMED_WAITING
  656. at java.lang.Object.wait(Native Method)
  657. - waiting on <4a1a424c> (a org.apache.hadoop.ipc.Client$Connection)
  658. at org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:924)
  659. at org.apache.hadoop.ipc.Client$Connection.run(Client.java:969)
  660.  
  661. Locked ownable synchronizers:
  662. - None
  663.  
  664. "IPC Client (1414013111) connection to ip-10-99-154-126.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.126:60020 from hbase" - Thread t@3083166
  665. java.lang.Thread.State: TIMED_WAITING
  666. at java.lang.Object.wait(Native Method)
  667. - waiting on <6b56128a> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  668. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  669. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  670.  
  671. Locked ownable synchronizers:
  672. - None
  673.  
  674. "IPC Client (1414013111) connection to ip-10-99-154-165.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.165:60020 from hbase" - Thread t@3083163
  675. java.lang.Thread.State: TIMED_WAITING
  676. at java.lang.Object.wait(Native Method)
  677. - waiting on <31199cb> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  678. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  679. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  680.  
  681. Locked ownable synchronizers:
  682. - None
  683.  
  684. "IPC Client (1414013111) connection to ip-10-99-154-152.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.152:60020 from hbase" - Thread t@3083153
  685. java.lang.Thread.State: TIMED_WAITING
  686. at java.lang.Object.wait(Native Method)
  687. - waiting on <408fcb7b> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  688. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  689. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  690.  
  691. Locked ownable synchronizers:
  692. - None
  693.  
  694. "IPC Client (1414013111) connection to ip-10-99-154-115.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.115:60020 from hbase" - Thread t@3083135
  695. java.lang.Thread.State: TIMED_WAITING
  696. at java.lang.Object.wait(Native Method)
  697. - waiting on <49c97cf5> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  698. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  699. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  700.  
  701. Locked ownable synchronizers:
  702. - None
  703.  
  704. "IPC Client (1414013111) connection to ip-10-99-154-118.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.118:60020 from hbase" - Thread t@2965213
  705. java.lang.Thread.State: TIMED_WAITING
  706. at java.lang.Object.wait(Native Method)
  707. - waiting on <7bbcbdae> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  708. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  709. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  710.  
  711. Locked ownable synchronizers:
  712. - None
  713.  
  714. "IPC Client (1414013111) connection to ip-10-99-154-141.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.141:60020 from hbase" - Thread t@2958930
  715. java.lang.Thread.State: TIMED_WAITING
  716. at java.lang.Object.wait(Native Method)
  717. - waiting on <2651ac86> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  718. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  719. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  720.  
  721. Locked ownable synchronizers:
  722. - None
  723.  
  724. "IPC Client (1414013111) connection to ip-10-99-154-151.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.151:60020 from hbase" - Thread t@2958082
  725. java.lang.Thread.State: TIMED_WAITING
  726. at java.lang.Object.wait(Native Method)
  727. - waiting on <2f08ac45> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  728. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  729. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  730.  
  731. Locked ownable synchronizers:
  732. - None
  733.  
  734. "IPC Client (1414013111) connection to ip-10-99-154-140.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.140:60020 from hbase" - Thread t@2956083
  735. java.lang.Thread.State: TIMED_WAITING
  736. at java.lang.Object.wait(Native Method)
  737. - waiting on <5102cfd4> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  738. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  739. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  740.  
  741. Locked ownable synchronizers:
  742. - None
  743.  
  744. "IPC Client (1414013111) connection to ip-10-99-154-124.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.124:60020 from hbase" - Thread t@2923847
  745. java.lang.Thread.State: TIMED_WAITING
  746. at java.lang.Object.wait(Native Method)
  747. - waiting on <31c8f5ad> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  748. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  749. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  750.  
  751. Locked ownable synchronizers:
  752. - None
  753.  
  754. "IPC Client (1414013111) connection to ip-10-99-154-130.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.130:60020 from hbase" - Thread t@2582705
  755. java.lang.Thread.State: TIMED_WAITING
  756. at java.lang.Object.wait(Native Method)
  757. - waiting on <1dfda8ee> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  758. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  759. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  760.  
  761. Locked ownable synchronizers:
  762. - None
  763.  
  764. "IPC Client (1414013111) connection to ip-10-99-154-58.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.58:60020 from hbase" - Thread t@2520942
  765. java.lang.Thread.State: TIMED_WAITING
  766. at java.lang.Object.wait(Native Method)
  767. - waiting on <755de18d> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  768. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  769. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  770.  
  771. Locked ownable synchronizers:
  772. - None
  773.  
  774. "IPC Client (1414013111) connection to ip-10-99-154-110.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.110:60020 from hbase" - Thread t@2518324
  775. java.lang.Thread.State: TIMED_WAITING
  776. at java.lang.Object.wait(Native Method)
  777. - waiting on <6d0e027c> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  778. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  779. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  780.  
  781. Locked ownable synchronizers:
  782. - None
  783.  
  784. "IPC Client (1414013111) connection to ip-10-99-154-53.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.53:60020 from hbase" - Thread t@2511447
  785. java.lang.Thread.State: TIMED_WAITING
  786. at java.lang.Object.wait(Native Method)
  787. - waiting on <5658fdbf> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  788. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  789. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  790.  
  791. Locked ownable synchronizers:
  792. - None
  793.  
  794. "IPC Client (1414013111) connection to ip-10-99-154-143.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.143:60020 from hbase" - Thread t@2507113
  795. java.lang.Thread.State: TIMED_WAITING
  796. at java.lang.Object.wait(Native Method)
  797. - waiting on <63cd0c81> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  798. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  799. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  800.  
  801. Locked ownable synchronizers:
  802. - None
  803.  
  804. "IPC Client (1414013111) connection to ip-10-99-154-104.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.104:60020 from hbase" - Thread t@2506996
  805. java.lang.Thread.State: TIMED_WAITING
  806. at java.lang.Object.wait(Native Method)
  807. - waiting on <7d50a5bb> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  808. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  809. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  810.  
  811. Locked ownable synchronizers:
  812. - None
  813.  
  814. "IPC Client (1414013111) connection to ip-10-99-154-156.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.156:60020 from hbase" - Thread t@2489113
  815. java.lang.Thread.State: TIMED_WAITING
  816. at java.lang.Object.wait(Native Method)
  817. - waiting on <2e0992be> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  818. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  819. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  820.  
  821. Locked ownable synchronizers:
  822. - None
  823.  
  824. "IPC Client (1414013111) connection to ip-10-99-154-168.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.168:60020 from hbase" - Thread t@2486092
  825. java.lang.Thread.State: TIMED_WAITING
  826. at java.lang.Object.wait(Native Method)
  827. - waiting on <1184e306> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  828. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  829. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  830.  
  831. Locked ownable synchronizers:
  832. - None
  833.  
  834. "IPC Client (1414013111) connection to ip-10-99-154-139.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.139:60020 from hbase" - Thread t@2455647
  835. java.lang.Thread.State: TIMED_WAITING
  836. at java.lang.Object.wait(Native Method)
  837. - waiting on <4a7ce347> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  838. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  839. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  840.  
  841. Locked ownable synchronizers:
  842. - None
  843.  
  844. "IPC Client (1414013111) connection to ip-10-99-154-154.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.154:60020 from hbase" - Thread t@2447662
  845. java.lang.Thread.State: TIMED_WAITING
  846. at java.lang.Object.wait(Native Method)
  847. - waiting on <4879a5a7> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  848. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  849. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  850.  
  851. Locked ownable synchronizers:
  852. - None
  853.  
  854. "IPC Client (1414013111) connection to ip-10-99-154-155.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.155:60020 from hbase" - Thread t@2446377
  855. java.lang.Thread.State: TIMED_WAITING
  856. at java.lang.Object.wait(Native Method)
  857. - waiting on <1107dc6d> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  858. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  859. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  860.  
  861. Locked ownable synchronizers:
  862. - None
  863.  
  864. "IPC Client (1414013111) connection to ip-10-99-154-128.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.128:60020 from hbase" - Thread t@2437786
  865. java.lang.Thread.State: TIMED_WAITING
  866. at java.lang.Object.wait(Native Method)
  867. - waiting on <75dc9ae3> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  868. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  869. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  870.  
  871. Locked ownable synchronizers:
  872. - None
  873.  
  874. "IPC Client (1414013111) connection to ip-10-99-154-119.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.119:60020 from hbase" - Thread t@2423126
  875. java.lang.Thread.State: TIMED_WAITING
  876. at java.lang.Object.wait(Native Method)
  877. - waiting on <22361df7> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  878. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  879. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  880.  
  881. Locked ownable synchronizers:
  882. - None
  883.  
  884. "IPC Client (1414013111) connection to ip-10-99-154-106.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.106:60020 from hbase" - Thread t@2421333
  885. java.lang.Thread.State: RUNNABLE
  886. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  887. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  888. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  889. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  890. - locked <16535cb2> (a sun.nio.ch.Util$2)
  891. - locked <45fb73b5> (a java.util.Collections$UnmodifiableSet)
  892. - locked <47314819> (a sun.nio.ch.EPollSelectorImpl)
  893. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  894. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335)
  895. at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157)
  896. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161)
  897. at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131)
  898. at java.io.FilterInputStream.read(FilterInputStream.java:133)
  899. at java.io.BufferedInputStream.fill(BufferedInputStream.java:246)
  900. at java.io.BufferedInputStream.read(BufferedInputStream.java:265)
  901. - locked <37d686c> (a java.io.BufferedInputStream)
  902. at java.io.DataInputStream.readInt(DataInputStream.java:387)
  903. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.readResponse(RpcClientImpl.java:934)
  904. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:554)
  905.  
  906. Locked ownable synchronizers:
  907. - None
  908.  
  909. "IPC Client (1414013111) connection to ip-10-99-154-153.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.153:60020 from hbase" - Thread t@2419107
  910. java.lang.Thread.State: TIMED_WAITING
  911. at java.lang.Object.wait(Native Method)
  912. - waiting on <3e0a1085> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  913. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  914. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  915.  
  916. Locked ownable synchronizers:
  917. - None
  918.  
  919. "IPC Client (1414013111) connection to ip-10-99-154-123.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.123:60020 from hbase" - Thread t@2417303
  920. java.lang.Thread.State: TIMED_WAITING
  921. at java.lang.Object.wait(Native Method)
  922. - waiting on <68f209c5> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  923. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  924. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  925.  
  926. Locked ownable synchronizers:
  927. - None
  928.  
  929. "IPC Client (1414013111) connection to ip-10-99-154-111.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.111:60020 from hbase" - Thread t@2377022
  930. java.lang.Thread.State: TIMED_WAITING
  931. at java.lang.Object.wait(Native Method)
  932. - waiting on <3b2a40b1> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  933. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  934. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  935.  
  936. Locked ownable synchronizers:
  937. - None
  938.  
  939. "IPC Client (1414013111) connection to ip-10-99-154-142.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.154.142:60020 from hbase" - Thread t@2365179
  940. java.lang.Thread.State: TIMED_WAITING
  941. at java.lang.Object.wait(Native Method)
  942. - waiting on <3041aab9> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  943. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  944. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  945.  
  946. Locked ownable synchronizers:
  947. - None
  948.  
  949. "RS_LOG_REPLAY_OPS-ip-10-99-181-146:60020-1" - Thread t@2344773
  950. java.lang.Thread.State: WAITING
  951. at sun.misc.Unsafe.park(Native Method)
  952. - parking to wait for <7baf7ba5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  953. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  954. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  955. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  956. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  957. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  958. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  959. at java.lang.Thread.run(Thread.java:745)
  960.  
  961. Locked ownable synchronizers:
  962. - None
  963.  
  964. "RS_CLOSE_REGION-ip-10-99-181-146:60020-2" - Thread t@532149
  965. java.lang.Thread.State: WAITING
  966. at sun.misc.Unsafe.park(Native Method)
  967. - parking to wait for <72ae2aa5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  968. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  969. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  970. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  971. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  972. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  973. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  974. at java.lang.Thread.run(Thread.java:745)
  975.  
  976. Locked ownable synchronizers:
  977. - None
  978.  
  979. "RS_CLOSE_REGION-ip-10-99-181-146:60020-1" - Thread t@520640
  980. java.lang.Thread.State: WAITING
  981. at sun.misc.Unsafe.park(Native Method)
  982. - parking to wait for <72ae2aa5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  983. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  984. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  985. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  986. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  987. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  988. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  989. at java.lang.Thread.run(Thread.java:745)
  990.  
  991. Locked ownable synchronizers:
  992. - None
  993.  
  994. "RS_CLOSE_REGION-ip-10-99-181-146:60020-0" - Thread t@520578
  995. java.lang.Thread.State: WAITING
  996. at sun.misc.Unsafe.park(Native Method)
  997. - parking to wait for <72ae2aa5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  998. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  999. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1000. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1001. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1002. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1003. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1004. at java.lang.Thread.run(Thread.java:745)
  1005.  
  1006. Locked ownable synchronizers:
  1007. - None
  1008.  
  1009. "RS_LOG_REPLAY_OPS-ip-10-99-181-146:60020-0" - Thread t@518201
  1010. java.lang.Thread.State: WAITING
  1011. at sun.misc.Unsafe.park(Native Method)
  1012. - parking to wait for <7baf7ba5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1013. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1014. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1015. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1016. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1017. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1018. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1019. at java.lang.Thread.run(Thread.java:745)
  1020.  
  1021. Locked ownable synchronizers:
  1022. - None
  1023.  
  1024. "ShortCircuitCache_SlotReleaser" - Thread t@4485
  1025. java.lang.Thread.State: WAITING
  1026. at sun.misc.Unsafe.park(Native Method)
  1027. - parking to wait for <6294c47a> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1028. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1029. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1030. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
  1031. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1032. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1033. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1034. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1035. at java.lang.Thread.run(Thread.java:745)
  1036.  
  1037. Locked ownable synchronizers:
  1038. - None
  1039.  
  1040. "RS_OPEN_REGION-ip-10-99-181-146:60020-2" - Thread t@1628
  1041. java.lang.Thread.State: WAITING
  1042. at sun.misc.Unsafe.park(Native Method)
  1043. - parking to wait for <4f371380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1044. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1045. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1046. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1047. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1048. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1049. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1050. at java.lang.Thread.run(Thread.java:745)
  1051.  
  1052. Locked ownable synchronizers:
  1053. - None
  1054.  
  1055. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-longCompactions-1480229289220" - Thread t@389
  1056. java.lang.Thread.State: WAITING
  1057. at sun.misc.Unsafe.park(Native Method)
  1058. - parking to wait for <4f703e43> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1059. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1060. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1061. at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549)
  1062. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1063. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1064. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1065. at java.lang.Thread.run(Thread.java:745)
  1066.  
  1067. Locked ownable synchronizers:
  1068. - None
  1069.  
  1070. "JMX server connection timeout 355" - Thread t@355
  1071. java.lang.Thread.State: TIMED_WAITING
  1072. at java.lang.Object.wait(Native Method)
  1073. - waiting on <51347b7a> (a [I)
  1074. at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(ServerCommunicatorAdmin.java:168)
  1075. at java.lang.Thread.run(Thread.java:745)
  1076.  
  1077. Locked ownable synchronizers:
  1078. - None
  1079.  
  1080. "RMI Scheduler(0)" - Thread t@353
  1081. java.lang.Thread.State: TIMED_WAITING
  1082. at sun.misc.Unsafe.park(Native Method)
  1083. - parking to wait for <716e8de2> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1084. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1085. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1086. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1087. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1088. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1089. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1090. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1091. at java.lang.Thread.run(Thread.java:745)
  1092.  
  1093. Locked ownable synchronizers:
  1094. - None
  1095.  
  1096. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-splits-1480229283126" - Thread t@263
  1097. java.lang.Thread.State: WAITING
  1098. at sun.misc.Unsafe.park(Native Method)
  1099. - parking to wait for <4d5ac187> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1100. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1101. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1102. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1103. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1104. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1105. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1106. at java.lang.Thread.run(Thread.java:745)
  1107.  
  1108. Locked ownable synchronizers:
  1109. - None
  1110.  
  1111. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-shortCompactions-1480229281547" - Thread t@242
  1112. java.lang.Thread.State: RUNNABLE
  1113. at org.apache.hadoop.hbase.io.encoding.FastDiffDeltaEncoder.compressSingleKeyValue(FastDiffDeltaEncoder.java:270)
  1114. at org.apache.hadoop.hbase.io.encoding.FastDiffDeltaEncoder.internalEncode(FastDiffDeltaEncoder.java:245)
  1115. at org.apache.hadoop.hbase.io.encoding.BufferedDataBlockEncoder.encode(BufferedDataBlockEncoder.java:987)
  1116. at org.apache.hadoop.hbase.io.encoding.FastDiffDeltaEncoder.encode(FastDiffDeltaEncoder.java:58)
  1117. at org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl.encode(HFileDataBlockEncoderImpl.java:97)
  1118. at org.apache.hadoop.hbase.io.hfile.HFileBlock$Writer.write(HFileBlock.java:866)
  1119. at org.apache.hadoop.hbase.io.hfile.HFileWriterV2.append(HFileWriterV2.java:270)
  1120. at org.apache.hadoop.hbase.io.hfile.HFileWriterV3.append(HFileWriterV3.java:87)
  1121. at org.apache.hadoop.hbase.regionserver.StoreFile$Writer.append(StoreFile.java:949)
  1122. at org.apache.hadoop.hbase.regionserver.compactions.Compactor.performCompaction(Compactor.java:282)
  1123. at org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:105)
  1124. at org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:124)
  1125. at org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1233)
  1126. at org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:1770)
  1127. at org.apache.hadoop.hbase.regionserver.CompactSplitThread$CompactionRunner.run(CompactSplitThread.java:520)
  1128. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1129. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1130. at java.lang.Thread.run(Thread.java:745)
  1131.  
  1132. Locked ownable synchronizers:
  1133. - locked <80dec9f> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  1134.  
  1135. "ShortCircuitCache_Cleaner" - Thread t@236
  1136. java.lang.Thread.State: TIMED_WAITING
  1137. at sun.misc.Unsafe.park(Native Method)
  1138. - parking to wait for <7a46e74d> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1139. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1140. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1141. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1142. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1143. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1144. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1145. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1146. at java.lang.Thread.run(Thread.java:745)
  1147.  
  1148. Locked ownable synchronizers:
  1149. - None
  1150.  
  1151. "RS_OPEN_REGION-ip-10-99-181-146:60020-1" - Thread t@231
  1152. java.lang.Thread.State: WAITING
  1153. at sun.misc.Unsafe.park(Native Method)
  1154. - parking to wait for <4f371380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1155. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1156. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1157. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1158. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1159. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1160. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1161. at java.lang.Thread.run(Thread.java:745)
  1162.  
  1163. Locked ownable synchronizers:
  1164. - None
  1165.  
  1166. "RS_OPEN_REGION-ip-10-99-181-146:60020-0" - Thread t@230
  1167. java.lang.Thread.State: WAITING
  1168. at sun.misc.Unsafe.park(Native Method)
  1169. - parking to wait for <4f371380> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1170. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1171. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1172. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1173. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1174. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1175. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1176. at java.lang.Thread.run(Thread.java:745)
  1177.  
  1178. Locked ownable synchronizers:
  1179. - None
  1180.  
  1181. "SplitLogWorker-ip-10-99-181-146:60020" - Thread t@228
  1182. java.lang.Thread.State: TIMED_WAITING
  1183. at java.lang.Object.wait(Native Method)
  1184. - waiting on <23f99d95> (a java.lang.Object)
  1185. at org.apache.hadoop.hbase.coordination.ZkSplitLogWorkerCoordination.taskLoop(ZkSplitLogWorkerCoordination.java:444)
  1186. at org.apache.hadoop.hbase.regionserver.SplitLogWorker.run(SplitLogWorker.java:142)
  1187. at java.lang.Thread.run(Thread.java:745)
  1188.  
  1189. Locked ownable synchronizers:
  1190. - None
  1191.  
  1192. "ip-10-99-181-146:60020Replication Statistics #0" - Thread t@227
  1193. java.lang.Thread.State: TIMED_WAITING
  1194. at sun.misc.Unsafe.park(Native Method)
  1195. - parking to wait for <3aa4df20> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1196. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1197. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1198. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1199. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1200. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1201. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1202. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1203. at java.lang.Thread.run(Thread.java:745)
  1204.  
  1205. Locked ownable synchronizers:
  1206. - None
  1207.  
  1208. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-EventThread" - Thread t@225
  1209. java.lang.Thread.State: WAITING
  1210. at sun.misc.Unsafe.park(Native Method)
  1211. - parking to wait for <77153bb6> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1212. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1213. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1214. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1215. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  1216.  
  1217. Locked ownable synchronizers:
  1218. - None
  1219.  
  1220. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-SendThread(ip-10-99-180-158.aolp-prd.us-east-1.ec2.aolcloud.net:2181)" - Thread t@224
  1221. java.lang.Thread.State: RUNNABLE
  1222. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1223. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  1224. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1225. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  1226. - locked <760c2b92> (a sun.nio.ch.Util$2)
  1227. - locked <7b83dc01> (a java.util.Collections$UnmodifiableSet)
  1228. - locked <2892b17> (a sun.nio.ch.EPollSelectorImpl)
  1229. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  1230. at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
  1231. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  1232.  
  1233. Locked ownable synchronizers:
  1234. - None
  1235.  
  1236. "ReplicationExecutor-0" - Thread t@223
  1237. java.lang.Thread.State: WAITING
  1238. at sun.misc.Unsafe.park(Native Method)
  1239. - parking to wait for <3c74cfb> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1240. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1241. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1242. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1243. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1244. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1245. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1246. at java.lang.Thread.run(Thread.java:745)
  1247.  
  1248. Locked ownable synchronizers:
  1249. - None
  1250.  
  1251. "org.apache.hadoop.hdfs.PeerCache@5d3fa336" - Thread t@221
  1252. java.lang.Thread.State: TIMED_WAITING
  1253. at java.lang.Thread.sleep(Native Method)
  1254. at org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:255)
  1255. at org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
  1256. at org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
  1257. at java.lang.Thread.run(Thread.java:745)
  1258.  
  1259. Locked ownable synchronizers:
  1260. - None
  1261.  
  1262. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020.replicationSource,1.replicationSource.ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net%2C60020%2C1480229275047.null0,1" - Thread t@219
  1263. java.lang.Thread.State: TIMED_WAITING
  1264. at java.lang.Object.wait(Native Method)
  1265. - waiting on <f47027d> (a org.apache.hadoop.hbase.ipc.Call)
  1266. at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1208)
  1267. at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:216)
  1268. at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:300)
  1269. at org.apache.hadoop.hbase.protobuf.generated.AdminProtos$AdminService$BlockingStub.replicateWALEntry(AdminProtos.java:21783)
  1270. at org.apache.hadoop.hbase.protobuf.ReplicationProtbufUtil.replicateWALEntry(ReplicationProtbufUtil.java:65)
  1271. at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.replicate(HBaseInterClusterReplicationEndpoint.java:162)
  1272. at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource$ReplicationSourceWorkerThread.shipEdits(ReplicationSource.java:907)
  1273. at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource$ReplicationSourceWorkerThread.run(ReplicationSource.java:617)
  1274.  
  1275. Locked ownable synchronizers:
  1276. - None
  1277.  
  1278. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020.replicationSource,1-EventThread" - Thread t@218
  1279. java.lang.Thread.State: WAITING
  1280. at sun.misc.Unsafe.park(Native Method)
  1281. - parking to wait for <c98dc7> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1282. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1283. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1284. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1285. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  1286.  
  1287. Locked ownable synchronizers:
  1288. - None
  1289.  
  1290. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020.replicationSource,1-SendThread(ip-10-99-154-137.aolp-prd.us-east-1.ec2.aolcloud.net:2181)" - Thread t@217
  1291. java.lang.Thread.State: RUNNABLE
  1292. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1293. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  1294. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1295. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  1296. - locked <2b3c8dcd> (a sun.nio.ch.Util$2)
  1297. - locked <3cfb6383> (a java.util.Collections$UnmodifiableSet)
  1298. - locked <59fdc3c9> (a sun.nio.ch.EPollSelectorImpl)
  1299. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  1300. at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
  1301. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  1302.  
  1303. Locked ownable synchronizers:
  1304. - None
  1305.  
  1306. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-EventThread" - Thread t@216
  1307. java.lang.Thread.State: WAITING
  1308. at sun.misc.Unsafe.park(Native Method)
  1309. - parking to wait for <6fb3ad7e> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1310. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1311. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1312. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1313. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  1314.  
  1315. Locked ownable synchronizers:
  1316. - None
  1317.  
  1318. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-SendThread(ip-10-99-154-137.aolp-prd.us-east-1.ec2.aolcloud.net:2181)" - Thread t@215
  1319. java.lang.Thread.State: RUNNABLE
  1320. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1321. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  1322. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1323. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  1324. - locked <12e79a72> (a sun.nio.ch.Util$2)
  1325. - locked <5d7a2f06> (a java.util.Collections$UnmodifiableSet)
  1326. - locked <3afaa851> (a sun.nio.ch.EPollSelectorImpl)
  1327. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  1328. at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
  1329. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  1330.  
  1331. Locked ownable synchronizers:
  1332. - None
  1333.  
  1334. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020.leaseChecker" - Thread t@193
  1335. java.lang.Thread.State: TIMED_WAITING
  1336. at java.lang.Thread.sleep(Native Method)
  1337. at org.apache.hadoop.hbase.regionserver.Leases.run(Leases.java:94)
  1338. at java.lang.Thread.run(Thread.java:745)
  1339.  
  1340. Locked ownable synchronizers:
  1341. - None
  1342.  
  1343. "ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net,60020,1480229275047_ChoreService_1" - Thread t@213
  1344. java.lang.Thread.State: TIMED_WAITING
  1345. at sun.misc.Unsafe.park(Native Method)
  1346. - parking to wait for <73722277> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1347. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1348. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1349. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1350. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1351. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1352. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1353. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1354. at java.lang.Thread.run(Thread.java:745)
  1355.  
  1356. Locked ownable synchronizers:
  1357. - None
  1358.  
  1359. "MemStoreFlusher.1" - Thread t@211
  1360. java.lang.Thread.State: TIMED_WAITING
  1361. at sun.misc.Unsafe.park(Native Method)
  1362. - parking to wait for <2d647f24> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1363. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1364. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1365. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)
  1366. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70)
  1367. at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:209)
  1368. at java.lang.Thread.run(Thread.java:745)
  1369.  
  1370. Locked ownable synchronizers:
  1371. - None
  1372.  
  1373. "MemStoreFlusher.0" - Thread t@209
  1374. java.lang.Thread.State: TIMED_WAITING
  1375. at sun.misc.Unsafe.park(Native Method)
  1376. - parking to wait for <2d647f24> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1377. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1378. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1379. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259)
  1380. at java.util.concurrent.DelayQueue.poll(DelayQueue.java:70)
  1381. at org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:209)
  1382. at java.lang.Thread.run(Thread.java:745)
  1383.  
  1384. Locked ownable synchronizers:
  1385. - None
  1386.  
  1387. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020.logRoller" - Thread t@189
  1388. java.lang.Thread.State: TIMED_WAITING
  1389. at java.lang.Object.wait(Native Method)
  1390. - waiting on <618f68fa> (a java.util.concurrent.atomic.AtomicBoolean)
  1391. at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:126)
  1392. at java.lang.Thread.run(Thread.java:745)
  1393.  
  1394. Locked ownable synchronizers:
  1395. - None
  1396.  
  1397. "HBase-Metrics2-1" - Thread t@208
  1398. java.lang.Thread.State: TIMED_WAITING
  1399. at sun.misc.Unsafe.park(Native Method)
  1400. - parking to wait for <13a5e1e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1401. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  1402. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  1403. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  1404. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  1405. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  1406. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  1407. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1408. at java.lang.Thread.run(Thread.java:745)
  1409.  
  1410. Locked ownable synchronizers:
  1411. - None
  1412.  
  1413. "sync.4" - Thread t@206
  1414. java.lang.Thread.State: TIMED_WAITING
  1415. at java.lang.Object.wait(Native Method)
  1416. - waiting on <67ba892d> (a java.util.LinkedList)
  1417. at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:2337)
  1418. at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:2224)
  1419. at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:2116)
  1420. at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:130)
  1421. at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.sync(ProtobufLogWriter.java:173)
  1422. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1379)
  1423. at java.lang.Thread.run(Thread.java:745)
  1424.  
  1425. Locked ownable synchronizers:
  1426. - None
  1427.  
  1428. "sync.3" - Thread t@205
  1429. java.lang.Thread.State: TIMED_WAITING
  1430. at java.lang.Object.wait(Native Method)
  1431. - waiting on <67ba892d> (a java.util.LinkedList)
  1432. at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:2337)
  1433. at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:2224)
  1434. at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:2116)
  1435. at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:130)
  1436. at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.sync(ProtobufLogWriter.java:173)
  1437. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1379)
  1438. at java.lang.Thread.run(Thread.java:745)
  1439.  
  1440. Locked ownable synchronizers:
  1441. - None
  1442.  
  1443. "sync.2" - Thread t@204
  1444. java.lang.Thread.State: TIMED_WAITING
  1445. at java.lang.Object.wait(Native Method)
  1446. - waiting on <67ba892d> (a java.util.LinkedList)
  1447. at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:2337)
  1448. at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:2224)
  1449. at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:2116)
  1450. at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:130)
  1451. at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.sync(ProtobufLogWriter.java:173)
  1452. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1379)
  1453. at java.lang.Thread.run(Thread.java:745)
  1454.  
  1455. Locked ownable synchronizers:
  1456. - None
  1457.  
  1458. "sync.1" - Thread t@203
  1459. java.lang.Thread.State: TIMED_WAITING
  1460. at java.lang.Object.wait(Native Method)
  1461. - waiting on <67ba892d> (a java.util.LinkedList)
  1462. at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:2337)
  1463. at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:2224)
  1464. at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:2116)
  1465. at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:130)
  1466. at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.sync(ProtobufLogWriter.java:173)
  1467. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1379)
  1468. at java.lang.Thread.run(Thread.java:745)
  1469.  
  1470. Locked ownable synchronizers:
  1471. - None
  1472.  
  1473. "sync.0" - Thread t@202
  1474. java.lang.Thread.State: TIMED_WAITING
  1475. at java.lang.Object.wait(Native Method)
  1476. - waiting on <67ba892d> (a java.util.LinkedList)
  1477. at org.apache.hadoop.hdfs.DFSOutputStream.waitForAckedSeqno(DFSOutputStream.java:2337)
  1478. at org.apache.hadoop.hdfs.DFSOutputStream.flushOrSync(DFSOutputStream.java:2224)
  1479. at org.apache.hadoop.hdfs.DFSOutputStream.hflush(DFSOutputStream.java:2116)
  1480. at org.apache.hadoop.fs.FSDataOutputStream.hflush(FSDataOutputStream.java:130)
  1481. at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.sync(ProtobufLogWriter.java:173)
  1482. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:1379)
  1483. at java.lang.Thread.run(Thread.java:745)
  1484.  
  1485. Locked ownable synchronizers:
  1486. - None
  1487.  
  1488. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020.append-pool1-t1" - Thread t@207
  1489. java.lang.Thread.State: WAITING
  1490. at sun.misc.Unsafe.park(Native Method)
  1491. - parking to wait for <491061a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1492. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1493. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1494. at com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:45)
  1495. at com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:55)
  1496. at com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:123)
  1497. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  1498. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  1499. at java.lang.Thread.run(Thread.java:745)
  1500.  
  1501. Locked ownable synchronizers:
  1502. - locked <74a95b9f> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  1503.  
  1504. "LeaseRenewer:hbase@nameservice1" - Thread t@200
  1505. java.lang.Thread.State: TIMED_WAITING
  1506. at java.lang.Thread.sleep(Native Method)
  1507. at org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:444)
  1508. at org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71)
  1509. at org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:304)
  1510. at java.lang.Thread.run(Thread.java:745)
  1511.  
  1512. Locked ownable synchronizers:
  1513. - None
  1514.  
  1515. "IPC Client (1414013111) connection to ip-10-99-180-159.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.180.159:60000 from hbase" - Thread t@195
  1516. java.lang.Thread.State: TIMED_WAITING
  1517. at java.lang.Object.wait(Native Method)
  1518. - waiting on <73437622> (a org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection)
  1519. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.waitForWork(RpcClientImpl.java:538)
  1520. at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:553)
  1521.  
  1522. Locked ownable synchronizers:
  1523. - None
  1524.  
  1525. "JvmPauseMonitor" - Thread t@192
  1526. java.lang.Thread.State: TIMED_WAITING
  1527. at java.lang.Thread.sleep(Native Method)
  1528. at org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:159)
  1529. at java.lang.Thread.run(Thread.java:745)
  1530.  
  1531. Locked ownable synchronizers:
  1532. - None
  1533.  
  1534. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-EventThread" - Thread t@191
  1535. java.lang.Thread.State: WAITING
  1536. at sun.misc.Unsafe.park(Native Method)
  1537. - parking to wait for <3914148b> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1538. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1539. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1540. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1541. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  1542.  
  1543. Locked ownable synchronizers:
  1544. - None
  1545.  
  1546. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020-SendThread(ip-10-99-180-159.aolp-prd.us-east-1.ec2.aolcloud.net:2181)" - Thread t@190
  1547. java.lang.Thread.State: RUNNABLE
  1548. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1549. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  1550. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1551. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  1552. - locked <d2ed74c> (a sun.nio.ch.Util$2)
  1553. - locked <2334f891> (a java.util.Collections$UnmodifiableSet)
  1554. - locked <2f3af1ee> (a sun.nio.ch.EPollSelectorImpl)
  1555. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  1556. at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
  1557. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  1558.  
  1559. Locked ownable synchronizers:
  1560. - None
  1561.  
  1562. "regionserver/ip-10-99-181-146.aolp-prd.us-east-1.ec2.aolcloud.net/10.99.181.146:60020" - Thread t@29
  1563. java.lang.Thread.State: TIMED_WAITING
  1564. at java.lang.Object.wait(Native Method)
  1565. - waiting on <312c036a> (a java.lang.Object)
  1566. at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:92)
  1567. at org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:56)
  1568. at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:905)
  1569. at java.lang.Thread.run(Thread.java:745)
  1570.  
  1571. Locked ownable synchronizers:
  1572. - None
  1573.  
  1574. "Timer-0" - Thread t@188
  1575. java.lang.Thread.State: TIMED_WAITING
  1576. at java.lang.Object.wait(Native Method)
  1577. - waiting on <5f900b65> (a java.util.TaskQueue)
  1578. at java.util.TimerThread.mainLoop(Timer.java:552)
  1579. at java.util.TimerThread.run(Timer.java:505)
  1580.  
  1581. Locked ownable synchronizers:
  1582. - None
  1583.  
  1584. "223975178@qtp-1164622694-1 - Acceptor0 SelectChannelConnector@0.0.0.0:60030" - Thread t@187
  1585. java.lang.Thread.State: RUNNABLE
  1586. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  1587. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  1588. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  1589. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  1590. - locked <40ed7c78> (a sun.nio.ch.Util$2)
  1591. - locked <3b82479e> (a java.util.Collections$UnmodifiableSet)
  1592. - locked <3f76c00f> (a sun.nio.ch.EPollSelectorImpl)
  1593. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  1594. at org.mortbay.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:511)
  1595. at org.mortbay.io.nio.SelectorManager.doSelect(SelectorManager.java:193)
  1596. at org.mortbay.jetty.nio.SelectChannelConnector.accept(SelectChannelConnector.java:124)
  1597. at org.mortbay.jetty.AbstractConnector$Acceptor.run(AbstractConnector.java:708)
  1598. at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
  1599.  
  1600. Locked ownable synchronizers:
  1601. - None
  1602.  
  1603. "ReplicationRpcServer.handler=2,queue=0,port=60020" - Thread t@184
  1604. java.lang.Thread.State: WAITING
  1605. at sun.misc.Unsafe.park(Native Method)
  1606. - parking to wait for <378a94d3> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1607. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1608. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1609. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1610. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1611. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1612. at java.lang.Thread.run(Thread.java:745)
  1613.  
  1614. Locked ownable synchronizers:
  1615. - None
  1616.  
  1617. "ReplicationRpcServer.handler=1,queue=0,port=60020" - Thread t@183
  1618. java.lang.Thread.State: WAITING
  1619. at sun.misc.Unsafe.park(Native Method)
  1620. - parking to wait for <378a94d3> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1621. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1622. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1623. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1624. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1625. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1626. at java.lang.Thread.run(Thread.java:745)
  1627.  
  1628. Locked ownable synchronizers:
  1629. - None
  1630.  
  1631. "ReplicationRpcServer.handler=0,queue=0,port=60020" - Thread t@182
  1632. java.lang.Thread.State: WAITING
  1633. at sun.misc.Unsafe.park(Native Method)
  1634. - parking to wait for <378a94d3> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1635. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1636. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1637. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1638. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1639. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1640. at java.lang.Thread.run(Thread.java:745)
  1641.  
  1642. Locked ownable synchronizers:
  1643. - None
  1644.  
  1645. "PriorityRpcServer.handler=9,queue=1,port=60020" - Thread t@181
  1646. java.lang.Thread.State: WAITING
  1647. at sun.misc.Unsafe.park(Native Method)
  1648. - parking to wait for <2e237ee5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1649. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1650. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1651. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1652. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1653. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1654. at java.lang.Thread.run(Thread.java:745)
  1655.  
  1656. Locked ownable synchronizers:
  1657. - None
  1658.  
  1659. "PriorityRpcServer.handler=8,queue=0,port=60020" - Thread t@180
  1660. java.lang.Thread.State: WAITING
  1661. at sun.misc.Unsafe.park(Native Method)
  1662. - parking to wait for <7f450f18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1663. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1664. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1665. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1666. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1667. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1668. at java.lang.Thread.run(Thread.java:745)
  1669.  
  1670. Locked ownable synchronizers:
  1671. - None
  1672.  
  1673. "PriorityRpcServer.handler=7,queue=1,port=60020" - Thread t@179
  1674. java.lang.Thread.State: WAITING
  1675. at sun.misc.Unsafe.park(Native Method)
  1676. - parking to wait for <2e237ee5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1677. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1678. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1679. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1680. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1681. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1682. at java.lang.Thread.run(Thread.java:745)
  1683.  
  1684. Locked ownable synchronizers:
  1685. - None
  1686.  
  1687. "PriorityRpcServer.handler=6,queue=0,port=60020" - Thread t@178
  1688. java.lang.Thread.State: WAITING
  1689. at sun.misc.Unsafe.park(Native Method)
  1690. - parking to wait for <7f450f18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1691. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1692. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1693. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1694. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1695. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1696. at java.lang.Thread.run(Thread.java:745)
  1697.  
  1698. Locked ownable synchronizers:
  1699. - None
  1700.  
  1701. "PriorityRpcServer.handler=5,queue=1,port=60020" - Thread t@177
  1702. java.lang.Thread.State: WAITING
  1703. at sun.misc.Unsafe.park(Native Method)
  1704. - parking to wait for <2e237ee5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1705. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1706. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1707. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1708. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1709. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1710. at java.lang.Thread.run(Thread.java:745)
  1711.  
  1712. Locked ownable synchronizers:
  1713. - None
  1714.  
  1715. "PriorityRpcServer.handler=4,queue=0,port=60020" - Thread t@176
  1716. java.lang.Thread.State: WAITING
  1717. at sun.misc.Unsafe.park(Native Method)
  1718. - parking to wait for <7f450f18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1719. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1720. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1721. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1722. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1723. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1724. at java.lang.Thread.run(Thread.java:745)
  1725.  
  1726. Locked ownable synchronizers:
  1727. - None
  1728.  
  1729. "PriorityRpcServer.handler=3,queue=1,port=60020" - Thread t@175
  1730. java.lang.Thread.State: WAITING
  1731. at sun.misc.Unsafe.park(Native Method)
  1732. - parking to wait for <2e237ee5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1733. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1734. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1735. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1736. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1737. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1738. at java.lang.Thread.run(Thread.java:745)
  1739.  
  1740. Locked ownable synchronizers:
  1741. - None
  1742.  
  1743. "PriorityRpcServer.handler=2,queue=0,port=60020" - Thread t@174
  1744. java.lang.Thread.State: WAITING
  1745. at sun.misc.Unsafe.park(Native Method)
  1746. - parking to wait for <7f450f18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1747. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1748. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1749. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1750. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1751. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1752. at java.lang.Thread.run(Thread.java:745)
  1753.  
  1754. Locked ownable synchronizers:
  1755. - None
  1756.  
  1757. "PriorityRpcServer.handler=1,queue=1,port=60020" - Thread t@173
  1758. java.lang.Thread.State: WAITING
  1759. at sun.misc.Unsafe.park(Native Method)
  1760. - parking to wait for <2e237ee5> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1761. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1762. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1763. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1764. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1765. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1766. at java.lang.Thread.run(Thread.java:745)
  1767.  
  1768. Locked ownable synchronizers:
  1769. - None
  1770.  
  1771. "PriorityRpcServer.handler=0,queue=0,port=60020" - Thread t@172
  1772. java.lang.Thread.State: WAITING
  1773. at sun.misc.Unsafe.park(Native Method)
  1774. - parking to wait for <7f450f18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  1775. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  1776. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  1777. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  1778. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:127)
  1779. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1780. at java.lang.Thread.run(Thread.java:745)
  1781.  
  1782. Locked ownable synchronizers:
  1783. - None
  1784.  
  1785. "B.defaultRpcServer.handler=119,queue=11,port=60020" - Thread t@171
  1786. java.lang.Thread.State: TIMED_WAITING
  1787. at java.lang.Object.wait(Native Method)
  1788. - waiting on <40f0ed24> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1789. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1790. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  1791. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  1792. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  1793. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  1794. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  1795. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  1796. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  1797. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  1798. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  1799. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  1800. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1801. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1802. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1803. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1804. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1805. at java.lang.Thread.run(Thread.java:745)
  1806.  
  1807. Locked ownable synchronizers:
  1808. - None
  1809.  
  1810. "B.defaultRpcServer.handler=118,queue=10,port=60020" - Thread t@170
  1811. java.lang.Thread.State: TIMED_WAITING
  1812. at java.lang.Object.wait(Native Method)
  1813. - waiting on <51654b83> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1814. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1815. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  1816. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  1817. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  1818. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  1819. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  1820. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  1821. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  1822. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  1823. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  1824. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  1825. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1826. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1827. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1828. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1829. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1830. at java.lang.Thread.run(Thread.java:745)
  1831.  
  1832. Locked ownable synchronizers:
  1833. - None
  1834.  
  1835. "B.defaultRpcServer.handler=117,queue=9,port=60020" - Thread t@169
  1836. java.lang.Thread.State: TIMED_WAITING
  1837. at java.lang.Object.wait(Native Method)
  1838. - waiting on <682e50e3> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1839. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1840. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  1841. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  1842. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  1843. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  1844. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  1845. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  1846. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  1847. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  1848. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  1849. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  1850. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  1851. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1852. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1853. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1854. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1855. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1856. at java.lang.Thread.run(Thread.java:745)
  1857.  
  1858. Locked ownable synchronizers:
  1859. - None
  1860.  
  1861. "B.defaultRpcServer.handler=116,queue=8,port=60020" - Thread t@168
  1862. java.lang.Thread.State: WAITING
  1863. at java.lang.Object.wait(Native Method)
  1864. - waiting on <54ecd24c> (a java.util.LinkedList)
  1865. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  1866. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  1867. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  1868. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  1869. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  1870. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1871. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1872. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1873. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1874. at java.lang.Thread.run(Thread.java:745)
  1875.  
  1876. Locked ownable synchronizers:
  1877. - None
  1878.  
  1879. "B.defaultRpcServer.handler=115,queue=7,port=60020" - Thread t@167
  1880. java.lang.Thread.State: TIMED_WAITING
  1881. at java.lang.Object.wait(Native Method)
  1882. - waiting on <5f20e7d> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1883. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1884. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  1885. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  1886. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  1887. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  1888. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  1889. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  1890. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  1891. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  1892. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  1893. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  1894. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1895. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1896. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1897. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1898. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1899. at java.lang.Thread.run(Thread.java:745)
  1900.  
  1901. Locked ownable synchronizers:
  1902. - None
  1903.  
  1904. "B.defaultRpcServer.handler=114,queue=6,port=60020" - Thread t@166
  1905. java.lang.Thread.State: TIMED_WAITING
  1906. at java.lang.Object.wait(Native Method)
  1907. - waiting on <2f27290a> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1908. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1909. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  1910. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  1911. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  1912. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  1913. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  1914. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  1915. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  1916. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  1917. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  1918. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  1919. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1920. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1921. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1922. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1923. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1924. at java.lang.Thread.run(Thread.java:745)
  1925.  
  1926. Locked ownable synchronizers:
  1927. - None
  1928.  
  1929. "B.defaultRpcServer.handler=113,queue=5,port=60020" - Thread t@165
  1930. java.lang.Thread.State: WAITING
  1931. at java.lang.Object.wait(Native Method)
  1932. - waiting on <af18ed8> (a java.util.LinkedList)
  1933. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  1934. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  1935. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  1936. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  1937. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  1938. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1939. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1940. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1941. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1942. at java.lang.Thread.run(Thread.java:745)
  1943.  
  1944. Locked ownable synchronizers:
  1945. - None
  1946.  
  1947. "B.defaultRpcServer.handler=112,queue=4,port=60020" - Thread t@164
  1948. java.lang.Thread.State: WAITING
  1949. at java.lang.Object.wait(Native Method)
  1950. - waiting on <7ef82fe0> (a java.util.LinkedList)
  1951. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  1952. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  1953. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  1954. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  1955. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  1956. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1957. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1958. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1959. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1960. at java.lang.Thread.run(Thread.java:745)
  1961.  
  1962. Locked ownable synchronizers:
  1963. - None
  1964.  
  1965. "B.defaultRpcServer.handler=111,queue=3,port=60020" - Thread t@163
  1966. java.lang.Thread.State: TIMED_WAITING
  1967. at java.lang.Object.wait(Native Method)
  1968. - waiting on <7715da8c> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1969. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1970. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  1971. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  1972. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  1973. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  1974. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  1975. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  1976. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  1977. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  1978. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  1979. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  1980. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  1981. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  1982. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  1983. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  1984. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  1985. at java.lang.Thread.run(Thread.java:745)
  1986.  
  1987. Locked ownable synchronizers:
  1988. - None
  1989.  
  1990. "B.defaultRpcServer.handler=110,queue=2,port=60020" - Thread t@162
  1991. java.lang.Thread.State: TIMED_WAITING
  1992. at java.lang.Object.wait(Native Method)
  1993. - waiting on <69713327> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  1994. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  1995. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  1996. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  1997. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  1998. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  1999. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2000. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2001. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2002. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2003. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2004. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2005. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2006. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2007. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2008. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2009. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2010. at java.lang.Thread.run(Thread.java:745)
  2011.  
  2012. Locked ownable synchronizers:
  2013. - None
  2014.  
  2015. "B.defaultRpcServer.handler=109,queue=1,port=60020" - Thread t@161
  2016. java.lang.Thread.State: WAITING
  2017. at java.lang.Object.wait(Native Method)
  2018. - waiting on <7205131a> (a java.util.LinkedList)
  2019. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2020. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2021. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2022. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2023. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2024. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2025. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2026. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2027. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2028. at java.lang.Thread.run(Thread.java:745)
  2029.  
  2030. Locked ownable synchronizers:
  2031. - None
  2032.  
  2033. "B.defaultRpcServer.handler=108,queue=0,port=60020" - Thread t@160
  2034. java.lang.Thread.State: TIMED_WAITING
  2035. at java.lang.Object.wait(Native Method)
  2036. - waiting on <6d592d69> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2037. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2038. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2039. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2040. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2041. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2042. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  2043. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  2044. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  2045. at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3442)
  2046. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3343)
  2047. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2048. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2049. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2050. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2051. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2052. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2053. at java.lang.Thread.run(Thread.java:745)
  2054.  
  2055. Locked ownable synchronizers:
  2056. - None
  2057.  
  2058. "B.defaultRpcServer.handler=107,queue=11,port=60020" - Thread t@159
  2059. java.lang.Thread.State: TIMED_WAITING
  2060. at java.lang.Object.wait(Native Method)
  2061. - waiting on <cb78dd2> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2062. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2063. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2064. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2065. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2066. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2067. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  2068. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  2069. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  2070. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2071. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  2072. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  2073. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  2074. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2075. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2076. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2077. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2078. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2079. at java.lang.Thread.run(Thread.java:745)
  2080.  
  2081. Locked ownable synchronizers:
  2082. - None
  2083.  
  2084. "B.defaultRpcServer.handler=106,queue=10,port=60020" - Thread t@158
  2085. java.lang.Thread.State: TIMED_WAITING
  2086. at java.lang.Object.wait(Native Method)
  2087. - waiting on <5f9cf8ee> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2088. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2089. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2090. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2091. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2092. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2093. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2094. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2095. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2096. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2097. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2098. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2099. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2100. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2101. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2102. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2103. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2104. at java.lang.Thread.run(Thread.java:745)
  2105.  
  2106. Locked ownable synchronizers:
  2107. - None
  2108.  
  2109. "B.defaultRpcServer.handler=105,queue=9,port=60020" - Thread t@157
  2110. java.lang.Thread.State: WAITING
  2111. at java.lang.Object.wait(Native Method)
  2112. - waiting on <7ffed664> (a java.util.LinkedList)
  2113. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2114. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2115. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2116. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2117. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2118. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2119. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2120. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2121. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2122. at java.lang.Thread.run(Thread.java:745)
  2123.  
  2124. Locked ownable synchronizers:
  2125. - None
  2126.  
  2127. "B.defaultRpcServer.handler=104,queue=8,port=60020" - Thread t@156
  2128. java.lang.Thread.State: TIMED_WAITING
  2129. at java.lang.Object.wait(Native Method)
  2130. - waiting on <61728fc8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2131. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2132. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2133. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2134. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2135. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2136. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  2137. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  2138. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  2139. at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3442)
  2140. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3343)
  2141. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2142. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2143. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2144. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2145. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2146. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2147. at java.lang.Thread.run(Thread.java:745)
  2148.  
  2149. Locked ownable synchronizers:
  2150. - None
  2151.  
  2152. "B.defaultRpcServer.handler=103,queue=7,port=60020" - Thread t@155
  2153. java.lang.Thread.State: TIMED_WAITING
  2154. at java.lang.Object.wait(Native Method)
  2155. - waiting on <6ab54ea3> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2156. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2157. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2158. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2159. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2160. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2161. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2162. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2163. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2164. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2165. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2166. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2167. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2168. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2169. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2170. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2171. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2172. at java.lang.Thread.run(Thread.java:745)
  2173.  
  2174. Locked ownable synchronizers:
  2175. - None
  2176.  
  2177. "B.defaultRpcServer.handler=102,queue=6,port=60020" - Thread t@154
  2178. java.lang.Thread.State: WAITING
  2179. at java.lang.Object.wait(Native Method)
  2180. - waiting on <369ab690> (a java.util.LinkedList)
  2181. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2182. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2183. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2184. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2185. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2186. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2187. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2188. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2189. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2190. at java.lang.Thread.run(Thread.java:745)
  2191.  
  2192. Locked ownable synchronizers:
  2193. - None
  2194.  
  2195. "B.defaultRpcServer.handler=101,queue=5,port=60020" - Thread t@153
  2196. java.lang.Thread.State: WAITING
  2197. at java.lang.Object.wait(Native Method)
  2198. - waiting on <7ffed664> (a java.util.LinkedList)
  2199. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2200. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2201. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2202. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2203. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2204. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2205. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2206. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2207. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2208. at java.lang.Thread.run(Thread.java:745)
  2209.  
  2210. Locked ownable synchronizers:
  2211. - None
  2212.  
  2213. "B.defaultRpcServer.handler=100,queue=4,port=60020" - Thread t@152
  2214. java.lang.Thread.State: TIMED_WAITING
  2215. at java.lang.Object.wait(Native Method)
  2216. - waiting on <174fe319> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2217. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2218. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2219. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2220. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2221. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2222. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2223. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2224. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2225. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2226. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2227. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2228. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2229. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2230. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2231. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2232. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2233. at java.lang.Thread.run(Thread.java:745)
  2234.  
  2235. Locked ownable synchronizers:
  2236. - None
  2237.  
  2238. "B.defaultRpcServer.handler=99,queue=3,port=60020" - Thread t@151
  2239. java.lang.Thread.State: TIMED_WAITING
  2240. at java.lang.Object.wait(Native Method)
  2241. - waiting on <d7c005b> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2242. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2243. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2244. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2245. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2246. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2247. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2248. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2249. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2250. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2251. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2252. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2253. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2254. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2255. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2256. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2257. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2258. at java.lang.Thread.run(Thread.java:745)
  2259.  
  2260. Locked ownable synchronizers:
  2261. - None
  2262.  
  2263. "B.defaultRpcServer.handler=98,queue=2,port=60020" - Thread t@150
  2264. java.lang.Thread.State: WAITING
  2265. at java.lang.Object.wait(Native Method)
  2266. - waiting on <7ffed664> (a java.util.LinkedList)
  2267. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2268. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2269. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2270. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2271. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2272. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2273. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2274. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2275. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2276. at java.lang.Thread.run(Thread.java:745)
  2277.  
  2278. Locked ownable synchronizers:
  2279. - None
  2280.  
  2281. "B.defaultRpcServer.handler=97,queue=1,port=60020" - Thread t@149
  2282. java.lang.Thread.State: TIMED_WAITING
  2283. at java.lang.Object.wait(Native Method)
  2284. - waiting on <3de61b4c> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2285. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2286. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2287. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2288. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2289. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2290. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2291. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2292. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2293. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2294. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2295. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2296. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2297. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2298. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2299. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2300. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2301. at java.lang.Thread.run(Thread.java:745)
  2302.  
  2303. Locked ownable synchronizers:
  2304. - None
  2305.  
  2306. "B.defaultRpcServer.handler=96,queue=0,port=60020" - Thread t@148
  2307. java.lang.Thread.State: TIMED_WAITING
  2308. at java.lang.Object.wait(Native Method)
  2309. - waiting on <2df91538> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2310. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2311. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2312. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2313. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2314. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2315. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  2316. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  2317. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  2318. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2319. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  2320. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  2321. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  2322. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2323. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2324. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2325. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2326. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2327. at java.lang.Thread.run(Thread.java:745)
  2328.  
  2329. Locked ownable synchronizers:
  2330. - None
  2331.  
  2332. "B.defaultRpcServer.handler=95,queue=11,port=60020" - Thread t@147
  2333. java.lang.Thread.State: TIMED_WAITING
  2334. at java.lang.Object.wait(Native Method)
  2335. - waiting on <4a09f26d> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2336. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2337. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2338. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2339. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2340. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2341. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2342. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2343. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2344. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2345. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2346. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2347. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2348. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2349. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2350. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2351. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2352. at java.lang.Thread.run(Thread.java:745)
  2353.  
  2354. Locked ownable synchronizers:
  2355. - None
  2356.  
  2357. "B.defaultRpcServer.handler=94,queue=10,port=60020" - Thread t@146
  2358. java.lang.Thread.State: TIMED_WAITING
  2359. at java.lang.Object.wait(Native Method)
  2360. - waiting on <71cd5d0a> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2361. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2362. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2363. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2364. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2365. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2366. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2367. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2368. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2369. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2370. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2371. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2372. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2373. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2374. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2375. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2376. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2377. at java.lang.Thread.run(Thread.java:745)
  2378.  
  2379. Locked ownable synchronizers:
  2380. - None
  2381.  
  2382. "B.defaultRpcServer.handler=93,queue=9,port=60020" - Thread t@145
  2383. java.lang.Thread.State: WAITING
  2384. at java.lang.Object.wait(Native Method)
  2385. - waiting on <3e1be0a9> (a java.util.LinkedList)
  2386. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2387. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2388. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2389. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2390. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2391. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2392. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2393. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2394. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2395. at java.lang.Thread.run(Thread.java:745)
  2396.  
  2397. Locked ownable synchronizers:
  2398. - None
  2399.  
  2400. "B.defaultRpcServer.handler=92,queue=8,port=60020" - Thread t@144
  2401. java.lang.Thread.State: WAITING
  2402. at java.lang.Object.wait(Native Method)
  2403. - waiting on <7ffed664> (a java.util.LinkedList)
  2404. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2405. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2406. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2407. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2408. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2409. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2410. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2411. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2412. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2413. at java.lang.Thread.run(Thread.java:745)
  2414.  
  2415. Locked ownable synchronizers:
  2416. - None
  2417.  
  2418. "B.defaultRpcServer.handler=91,queue=7,port=60020" - Thread t@143
  2419. java.lang.Thread.State: TIMED_WAITING
  2420. at java.lang.Object.wait(Native Method)
  2421. - waiting on <116f5509> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2422. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2423. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2424. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2425. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2426. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2427. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2428. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2429. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2430. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2431. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2432. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2433. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2434. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2435. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2436. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2437. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2438. at java.lang.Thread.run(Thread.java:745)
  2439.  
  2440. Locked ownable synchronizers:
  2441. - None
  2442.  
  2443. "B.defaultRpcServer.handler=90,queue=6,port=60020" - Thread t@142
  2444. java.lang.Thread.State: WAITING
  2445. at java.lang.Object.wait(Native Method)
  2446. - waiting on <5ec7f6fe> (a java.util.LinkedList)
  2447. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2448. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2449. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2450. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2451. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2452. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2453. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2454. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2455. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2456. at java.lang.Thread.run(Thread.java:745)
  2457.  
  2458. Locked ownable synchronizers:
  2459. - None
  2460.  
  2461. "B.defaultRpcServer.handler=89,queue=5,port=60020" - Thread t@141
  2462. java.lang.Thread.State: WAITING
  2463. at java.lang.Object.wait(Native Method)
  2464. - waiting on <1d65c4d1> (a java.util.LinkedList)
  2465. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2466. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2467. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2468. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2469. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2470. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2471. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2472. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2473. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2474. at java.lang.Thread.run(Thread.java:745)
  2475.  
  2476. Locked ownable synchronizers:
  2477. - None
  2478.  
  2479. "B.defaultRpcServer.handler=88,queue=4,port=60020" - Thread t@140
  2480. java.lang.Thread.State: TIMED_WAITING
  2481. at java.lang.Object.wait(Native Method)
  2482. - waiting on <1c3f5e16> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2483. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2484. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2485. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2486. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2487. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2488. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  2489. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  2490. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  2491. at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3442)
  2492. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3343)
  2493. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2494. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2495. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2496. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2497. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2498. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2499. at java.lang.Thread.run(Thread.java:745)
  2500.  
  2501. Locked ownable synchronizers:
  2502. - None
  2503.  
  2504. "B.defaultRpcServer.handler=87,queue=3,port=60020" - Thread t@139
  2505. java.lang.Thread.State: WAITING
  2506. at java.lang.Object.wait(Native Method)
  2507. - waiting on <5ec7f6fe> (a java.util.LinkedList)
  2508. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2509. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2510. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2511. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2512. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2513. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2514. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2515. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2516. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2517. at java.lang.Thread.run(Thread.java:745)
  2518.  
  2519. Locked ownable synchronizers:
  2520. - None
  2521.  
  2522. "B.defaultRpcServer.handler=86,queue=2,port=60020" - Thread t@138
  2523. java.lang.Thread.State: WAITING
  2524. at java.lang.Object.wait(Native Method)
  2525. - waiting on <5ec7f6fe> (a java.util.LinkedList)
  2526. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2527. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2528. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2529. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2530. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2531. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2532. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2533. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2534. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2535. at java.lang.Thread.run(Thread.java:745)
  2536.  
  2537. Locked ownable synchronizers:
  2538. - None
  2539.  
  2540. "B.defaultRpcServer.handler=85,queue=1,port=60020" - Thread t@137
  2541. java.lang.Thread.State: TIMED_WAITING
  2542. at java.lang.Object.wait(Native Method)
  2543. - waiting on <5ed8647> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2544. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2545. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2546. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2547. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2548. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2549. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2550. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2551. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2552. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2553. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2554. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2555. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2556. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2557. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2558. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2559. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2560. at java.lang.Thread.run(Thread.java:745)
  2561.  
  2562. Locked ownable synchronizers:
  2563. - None
  2564.  
  2565. "B.defaultRpcServer.handler=84,queue=0,port=60020" - Thread t@136
  2566. java.lang.Thread.State: WAITING
  2567. at java.lang.Object.wait(Native Method)
  2568. - waiting on <1d65c4d1> (a java.util.LinkedList)
  2569. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2570. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2571. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2572. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2573. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2574. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2575. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2576. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2577. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2578. at java.lang.Thread.run(Thread.java:745)
  2579.  
  2580. Locked ownable synchronizers:
  2581. - None
  2582.  
  2583. "B.defaultRpcServer.handler=83,queue=11,port=60020" - Thread t@135
  2584. java.lang.Thread.State: TIMED_WAITING
  2585. at java.lang.Object.wait(Native Method)
  2586. - waiting on <41cd4bfa> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2587. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2588. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2589. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2590. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2591. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2592. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  2593. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  2594. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  2595. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  2596. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  2597. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  2598. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  2599. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2600. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2601. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2602. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2603. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2604. at java.lang.Thread.run(Thread.java:745)
  2605.  
  2606. Locked ownable synchronizers:
  2607. - None
  2608.  
  2609. "B.defaultRpcServer.handler=82,queue=10,port=60020" - Thread t@134
  2610. java.lang.Thread.State: TIMED_WAITING
  2611. at java.lang.Object.wait(Native Method)
  2612. - waiting on <225dd275> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2613. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2614. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2615. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2616. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2617. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2618. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2619. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2620. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2621. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2622. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2623. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2624. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2625. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2626. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2627. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2628. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2629. at java.lang.Thread.run(Thread.java:745)
  2630.  
  2631. Locked ownable synchronizers:
  2632. - None
  2633.  
  2634. "B.defaultRpcServer.handler=81,queue=9,port=60020" - Thread t@133
  2635. java.lang.Thread.State: TIMED_WAITING
  2636. at java.lang.Object.wait(Native Method)
  2637. - waiting on <a8d3138> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2638. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2639. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2640. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2641. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2642. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2643. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2644. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2645. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2646. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2647. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2648. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2649. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2650. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2651. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2652. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2653. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2654. at java.lang.Thread.run(Thread.java:745)
  2655.  
  2656. Locked ownable synchronizers:
  2657. - None
  2658.  
  2659. "B.defaultRpcServer.handler=80,queue=8,port=60020" - Thread t@132
  2660. java.lang.Thread.State: WAITING
  2661. at java.lang.Object.wait(Native Method)
  2662. - waiting on <e96676a> (a java.util.LinkedList)
  2663. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2664. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2665. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2666. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2667. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2668. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2669. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2670. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2671. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2672. at java.lang.Thread.run(Thread.java:745)
  2673.  
  2674. Locked ownable synchronizers:
  2675. - None
  2676.  
  2677. "B.defaultRpcServer.handler=79,queue=7,port=60020" - Thread t@131
  2678. java.lang.Thread.State: TIMED_WAITING
  2679. at java.lang.Object.wait(Native Method)
  2680. - waiting on <15d541cf> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2681. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2682. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2683. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2684. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2685. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2686. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2687. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2688. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2689. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2690. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2691. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2692. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2693. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2694. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2695. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2696. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2697. at java.lang.Thread.run(Thread.java:745)
  2698.  
  2699. Locked ownable synchronizers:
  2700. - None
  2701.  
  2702. "B.defaultRpcServer.handler=78,queue=6,port=60020" - Thread t@130
  2703. java.lang.Thread.State: TIMED_WAITING
  2704. at java.lang.Object.wait(Native Method)
  2705. - waiting on <72267006> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2706. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2707. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2708. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2709. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2710. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2711. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2712. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2713. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2714. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2715. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2716. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2717. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2718. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2719. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2720. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2721. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2722. at java.lang.Thread.run(Thread.java:745)
  2723.  
  2724. Locked ownable synchronizers:
  2725. - None
  2726.  
  2727. "B.defaultRpcServer.handler=77,queue=5,port=60020" - Thread t@129
  2728. java.lang.Thread.State: WAITING
  2729. at java.lang.Object.wait(Native Method)
  2730. - waiting on <246752cd> (a java.util.LinkedList)
  2731. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2732. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2733. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2734. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2735. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2736. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2737. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2738. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2739. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2740. at java.lang.Thread.run(Thread.java:745)
  2741.  
  2742. Locked ownable synchronizers:
  2743. - None
  2744.  
  2745. "B.defaultRpcServer.handler=76,queue=4,port=60020" - Thread t@128
  2746. java.lang.Thread.State: TIMED_WAITING
  2747. at java.lang.Object.wait(Native Method)
  2748. - waiting on <4bec28f3> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2749. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2750. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2751. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2752. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2753. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2754. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2755. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2756. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2757. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2758. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2759. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2760. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2761. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2762. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2763. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2764. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2765. at java.lang.Thread.run(Thread.java:745)
  2766.  
  2767. Locked ownable synchronizers:
  2768. - None
  2769.  
  2770. "B.defaultRpcServer.handler=75,queue=3,port=60020" - Thread t@127
  2771. java.lang.Thread.State: TIMED_WAITING
  2772. at java.lang.Object.wait(Native Method)
  2773. - waiting on <313a0a2c> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2774. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2775. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2776. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2777. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2778. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2779. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2780. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2781. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2782. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2783. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2784. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2785. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2786. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2787. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2788. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2789. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2790. at java.lang.Thread.run(Thread.java:745)
  2791.  
  2792. Locked ownable synchronizers:
  2793. - None
  2794.  
  2795. "B.defaultRpcServer.handler=74,queue=2,port=60020" - Thread t@126
  2796. java.lang.Thread.State: TIMED_WAITING
  2797. at java.lang.Object.wait(Native Method)
  2798. - waiting on <7411afe3> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2799. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2800. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2801. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2802. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2803. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2804. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2805. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2806. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2807. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2808. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2809. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2810. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2811. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2812. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2813. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2814. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2815. at java.lang.Thread.run(Thread.java:745)
  2816.  
  2817. Locked ownable synchronizers:
  2818. - None
  2819.  
  2820. "B.defaultRpcServer.handler=73,queue=1,port=60020" - Thread t@125
  2821. java.lang.Thread.State: WAITING
  2822. at java.lang.Object.wait(Native Method)
  2823. - waiting on <4d90dfe5> (a java.util.LinkedList)
  2824. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2825. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2826. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2827. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2828. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2829. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2830. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2831. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2832. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2833. at java.lang.Thread.run(Thread.java:745)
  2834.  
  2835. Locked ownable synchronizers:
  2836. - None
  2837.  
  2838. "B.defaultRpcServer.handler=72,queue=0,port=60020" - Thread t@124
  2839. java.lang.Thread.State: TIMED_WAITING
  2840. at java.lang.Object.wait(Native Method)
  2841. - waiting on <36f56ee2> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2842. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2843. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2844. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2845. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2846. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2847. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2848. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2849. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2850. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2851. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2852. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2853. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2854. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2855. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2856. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2857. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2858. at java.lang.Thread.run(Thread.java:745)
  2859.  
  2860. Locked ownable synchronizers:
  2861. - None
  2862.  
  2863. "B.defaultRpcServer.handler=71,queue=11,port=60020" - Thread t@123
  2864. java.lang.Thread.State: TIMED_WAITING
  2865. at java.lang.Object.wait(Native Method)
  2866. - waiting on <557d458a> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2867. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2868. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2869. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2870. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2871. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2872. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2873. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2874. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2875. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2876. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2877. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2878. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2879. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2880. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2881. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2882. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2883. at java.lang.Thread.run(Thread.java:745)
  2884.  
  2885. Locked ownable synchronizers:
  2886. - None
  2887.  
  2888. "B.defaultRpcServer.handler=70,queue=10,port=60020" - Thread t@122
  2889. java.lang.Thread.State: WAITING
  2890. at java.lang.Object.wait(Native Method)
  2891. - waiting on <7205131a> (a java.util.LinkedList)
  2892. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2893. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2894. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2895. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2896. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2897. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2898. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2899. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2900. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2901. at java.lang.Thread.run(Thread.java:745)
  2902.  
  2903. Locked ownable synchronizers:
  2904. - None
  2905.  
  2906. "B.defaultRpcServer.handler=69,queue=9,port=60020" - Thread t@121
  2907. java.lang.Thread.State: TIMED_WAITING
  2908. at java.lang.Object.wait(Native Method)
  2909. - waiting on <34f75e63> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2910. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2911. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2912. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2913. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2914. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2915. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2916. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2917. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2918. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2919. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2920. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2921. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2922. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2923. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2924. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2925. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2926. at java.lang.Thread.run(Thread.java:745)
  2927.  
  2928. Locked ownable synchronizers:
  2929. - None
  2930.  
  2931. "B.defaultRpcServer.handler=68,queue=8,port=60020" - Thread t@120
  2932. java.lang.Thread.State: WAITING
  2933. at java.lang.Object.wait(Native Method)
  2934. - waiting on <7ffed664> (a java.util.LinkedList)
  2935. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  2936. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  2937. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  2938. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  2939. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  2940. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2941. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2942. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2943. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2944. at java.lang.Thread.run(Thread.java:745)
  2945.  
  2946. Locked ownable synchronizers:
  2947. - None
  2948.  
  2949. "B.defaultRpcServer.handler=67,queue=7,port=60020" - Thread t@119
  2950. java.lang.Thread.State: TIMED_WAITING
  2951. at java.lang.Object.wait(Native Method)
  2952. - waiting on <6df6caad> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2953. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2954. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2955. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2956. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2957. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2958. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2959. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2960. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2961. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2962. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2963. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2964. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2965. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2966. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2967. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2968. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2969. at java.lang.Thread.run(Thread.java:745)
  2970.  
  2971. Locked ownable synchronizers:
  2972. - None
  2973.  
  2974. "B.defaultRpcServer.handler=66,queue=6,port=60020" - Thread t@118
  2975. java.lang.Thread.State: TIMED_WAITING
  2976. at java.lang.Object.wait(Native Method)
  2977. - waiting on <582d7dac> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  2978. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  2979. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  2980. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  2981. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  2982. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  2983. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  2984. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  2985. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  2986. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  2987. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  2988. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  2989. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  2990. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  2991. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  2992. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  2993. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  2994. at java.lang.Thread.run(Thread.java:745)
  2995.  
  2996. Locked ownable synchronizers:
  2997. - None
  2998.  
  2999. "B.defaultRpcServer.handler=65,queue=5,port=60020" - Thread t@117
  3000. java.lang.Thread.State: WAITING
  3001. at java.lang.Object.wait(Native Method)
  3002. - waiting on <526870c2> (a java.util.LinkedList)
  3003. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3004. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3005. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3006. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3007. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3008. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3009. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3010. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3011. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3012. at java.lang.Thread.run(Thread.java:745)
  3013.  
  3014. Locked ownable synchronizers:
  3015. - None
  3016.  
  3017. "B.defaultRpcServer.handler=64,queue=4,port=60020" - Thread t@116
  3018. java.lang.Thread.State: TIMED_WAITING
  3019. at java.lang.Object.wait(Native Method)
  3020. - waiting on <32a6f8b7> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3021. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3022. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3023. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3024. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3025. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3026. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3027. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3028. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3029. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3030. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3031. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3032. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3033. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3034. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3035. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3036. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3037. at java.lang.Thread.run(Thread.java:745)
  3038.  
  3039. Locked ownable synchronizers:
  3040. - None
  3041.  
  3042. "B.defaultRpcServer.handler=63,queue=3,port=60020" - Thread t@115
  3043. java.lang.Thread.State: TIMED_WAITING
  3044. at java.lang.Object.wait(Native Method)
  3045. - waiting on <5298838c> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3046. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3047. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3048. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3049. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3050. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3051. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3052. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3053. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3054. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3055. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3056. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3057. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3058. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3059. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3060. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3061. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3062. at java.lang.Thread.run(Thread.java:745)
  3063.  
  3064. Locked ownable synchronizers:
  3065. - None
  3066.  
  3067. "B.defaultRpcServer.handler=62,queue=2,port=60020" - Thread t@114
  3068. java.lang.Thread.State: TIMED_WAITING
  3069. at java.lang.Object.wait(Native Method)
  3070. - waiting on <5c12cbfa> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3071. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3072. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3073. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3074. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3075. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3076. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3077. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3078. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3079. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3080. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3081. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3082. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3083. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3084. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3085. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3086. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3087. at java.lang.Thread.run(Thread.java:745)
  3088.  
  3089. Locked ownable synchronizers:
  3090. - None
  3091.  
  3092. "B.defaultRpcServer.handler=61,queue=1,port=60020" - Thread t@113
  3093. java.lang.Thread.State: TIMED_WAITING
  3094. at java.lang.Object.wait(Native Method)
  3095. - waiting on <c675617> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3096. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3097. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3098. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3099. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3100. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3101. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  3102. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  3103. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  3104. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  3105. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  3106. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  3107. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  3108. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3109. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3110. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3111. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3112. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3113. at java.lang.Thread.run(Thread.java:745)
  3114.  
  3115. Locked ownable synchronizers:
  3116. - None
  3117.  
  3118. "B.defaultRpcServer.handler=60,queue=0,port=60020" - Thread t@112
  3119. java.lang.Thread.State: TIMED_WAITING
  3120. at java.lang.Object.wait(Native Method)
  3121. - waiting on <571f4694> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3122. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3123. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3124. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3125. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3126. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3127. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3128. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3129. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3130. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3131. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3132. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3133. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3134. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3135. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3136. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3137. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3138. at java.lang.Thread.run(Thread.java:745)
  3139.  
  3140. Locked ownable synchronizers:
  3141. - None
  3142.  
  3143. "B.defaultRpcServer.handler=59,queue=11,port=60020" - Thread t@111
  3144. java.lang.Thread.State: WAITING
  3145. at java.lang.Object.wait(Native Method)
  3146. - waiting on <7ef82fe0> (a java.util.LinkedList)
  3147. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3148. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3149. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3150. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3151. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3152. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3153. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3154. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3155. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3156. at java.lang.Thread.run(Thread.java:745)
  3157.  
  3158. Locked ownable synchronizers:
  3159. - None
  3160.  
  3161. "B.defaultRpcServer.handler=58,queue=10,port=60020" - Thread t@110
  3162. java.lang.Thread.State: WAITING
  3163. at java.lang.Object.wait(Native Method)
  3164. - waiting on <7ffed664> (a java.util.LinkedList)
  3165. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3166. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3167. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3168. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3169. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3170. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3171. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3172. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3173. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3174. at java.lang.Thread.run(Thread.java:745)
  3175.  
  3176. Locked ownable synchronizers:
  3177. - None
  3178.  
  3179. "B.defaultRpcServer.handler=57,queue=9,port=60020" - Thread t@109
  3180. java.lang.Thread.State: TIMED_WAITING
  3181. at java.lang.Object.wait(Native Method)
  3182. - waiting on <2bfe41de> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3183. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3184. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3185. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3186. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3187. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3188. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  3189. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  3190. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  3191. at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3442)
  3192. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3343)
  3193. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3194. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3195. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3196. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3197. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3198. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3199. at java.lang.Thread.run(Thread.java:745)
  3200.  
  3201. Locked ownable synchronizers:
  3202. - None
  3203.  
  3204. "B.defaultRpcServer.handler=56,queue=8,port=60020" - Thread t@108
  3205. java.lang.Thread.State: TIMED_WAITING
  3206. at java.lang.Object.wait(Native Method)
  3207. - waiting on <35890c0c> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3208. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3209. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3210. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3211. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3212. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3213. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3214. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3215. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3216. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3217. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3218. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3219. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3220. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3221. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3222. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3223. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3224. at java.lang.Thread.run(Thread.java:745)
  3225.  
  3226. Locked ownable synchronizers:
  3227. - None
  3228.  
  3229. "B.defaultRpcServer.handler=55,queue=7,port=60020" - Thread t@107
  3230. java.lang.Thread.State: WAITING
  3231. at java.lang.Object.wait(Native Method)
  3232. - waiting on <246752cd> (a java.util.LinkedList)
  3233. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3234. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3235. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3236. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3237. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3238. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3239. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3240. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3241. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3242. at java.lang.Thread.run(Thread.java:745)
  3243.  
  3244. Locked ownable synchronizers:
  3245. - None
  3246.  
  3247. "B.defaultRpcServer.handler=54,queue=6,port=60020" - Thread t@106
  3248. java.lang.Thread.State: WAITING
  3249. at java.lang.Object.wait(Native Method)
  3250. - waiting on <26d7fb1d> (a java.util.LinkedList)
  3251. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3252. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3253. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3254. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3255. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3256. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3257. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3258. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3259. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3260. at java.lang.Thread.run(Thread.java:745)
  3261.  
  3262. Locked ownable synchronizers:
  3263. - None
  3264.  
  3265. "B.defaultRpcServer.handler=53,queue=5,port=60020" - Thread t@105
  3266. java.lang.Thread.State: WAITING
  3267. at java.lang.Object.wait(Native Method)
  3268. - waiting on <7ffed664> (a java.util.LinkedList)
  3269. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3270. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3271. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3272. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3273. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3274. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3275. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3276. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3277. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3278. at java.lang.Thread.run(Thread.java:745)
  3279.  
  3280. Locked ownable synchronizers:
  3281. - None
  3282.  
  3283. "B.defaultRpcServer.handler=52,queue=4,port=60020" - Thread t@104
  3284. java.lang.Thread.State: TIMED_WAITING
  3285. at java.lang.Object.wait(Native Method)
  3286. - waiting on <3a536cb1> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3287. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3288. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3289. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3290. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3291. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3292. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3293. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3294. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3295. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3296. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3297. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3298. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3299. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3300. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3301. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3302. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3303. at java.lang.Thread.run(Thread.java:745)
  3304.  
  3305. Locked ownable synchronizers:
  3306. - None
  3307.  
  3308. "B.defaultRpcServer.handler=51,queue=3,port=60020" - Thread t@103
  3309. java.lang.Thread.State: TIMED_WAITING
  3310. at java.lang.Object.wait(Native Method)
  3311. - waiting on <58b3e823> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3312. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3313. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3314. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3315. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3316. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3317. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3318. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3319. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3320. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3321. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3322. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3323. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3324. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3325. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3326. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3327. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3328. at java.lang.Thread.run(Thread.java:745)
  3329.  
  3330. Locked ownable synchronizers:
  3331. - None
  3332.  
  3333. "B.defaultRpcServer.handler=50,queue=2,port=60020" - Thread t@102
  3334. java.lang.Thread.State: WAITING
  3335. at java.lang.Object.wait(Native Method)
  3336. - waiting on <7205131a> (a java.util.LinkedList)
  3337. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3338. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3339. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3340. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3341. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3342. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3343. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3344. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3345. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3346. at java.lang.Thread.run(Thread.java:745)
  3347.  
  3348. Locked ownable synchronizers:
  3349. - None
  3350.  
  3351. "B.defaultRpcServer.handler=49,queue=1,port=60020" - Thread t@101
  3352. java.lang.Thread.State: TIMED_WAITING
  3353. at java.lang.Object.wait(Native Method)
  3354. - waiting on <7318f427> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3355. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3356. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3357. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3358. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3359. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3360. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3361. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3362. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3363. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3364. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3365. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3366. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3367. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3368. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3369. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3370. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3371. at java.lang.Thread.run(Thread.java:745)
  3372.  
  3373. Locked ownable synchronizers:
  3374. - None
  3375.  
  3376. "B.defaultRpcServer.handler=48,queue=0,port=60020" - Thread t@100
  3377. java.lang.Thread.State: TIMED_WAITING
  3378. at java.lang.Object.wait(Native Method)
  3379. - waiting on <1e1496e> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3380. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3381. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3382. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3383. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3384. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3385. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3386. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3387. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3388. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3389. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3390. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3391. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3392. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3393. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3394. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3395. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3396. at java.lang.Thread.run(Thread.java:745)
  3397.  
  3398. Locked ownable synchronizers:
  3399. - None
  3400.  
  3401. "B.defaultRpcServer.handler=47,queue=11,port=60020" - Thread t@99
  3402. java.lang.Thread.State: TIMED_WAITING
  3403. at java.lang.Object.wait(Native Method)
  3404. - waiting on <499e430e> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3405. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3406. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3407. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3408. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3409. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3410. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  3411. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  3412. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  3413. at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3442)
  3414. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3343)
  3415. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3416. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3417. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3418. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3419. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3420. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3421. at java.lang.Thread.run(Thread.java:745)
  3422.  
  3423. Locked ownable synchronizers:
  3424. - None
  3425.  
  3426. "B.defaultRpcServer.handler=46,queue=10,port=60020" - Thread t@98
  3427. java.lang.Thread.State: TIMED_WAITING
  3428. at java.lang.Object.wait(Native Method)
  3429. - waiting on <2daae290> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3430. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3431. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3432. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3433. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3434. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3435. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3436. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3437. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3438. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3439. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3440. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3441. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3442. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3443. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3444. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3445. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3446. at java.lang.Thread.run(Thread.java:745)
  3447.  
  3448. Locked ownable synchronizers:
  3449. - None
  3450.  
  3451. "B.defaultRpcServer.handler=45,queue=9,port=60020" - Thread t@97
  3452. java.lang.Thread.State: TIMED_WAITING
  3453. at java.lang.Object.wait(Native Method)
  3454. - waiting on <b0a65d8> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3455. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3456. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3457. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3458. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3459. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3460. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3461. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3462. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3463. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3464. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3465. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3466. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3467. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3468. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3469. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3470. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3471. at java.lang.Thread.run(Thread.java:745)
  3472.  
  3473. Locked ownable synchronizers:
  3474. - None
  3475.  
  3476. "B.defaultRpcServer.handler=44,queue=8,port=60020" - Thread t@96
  3477. java.lang.Thread.State: WAITING
  3478. at java.lang.Object.wait(Native Method)
  3479. - waiting on <968530f> (a java.util.LinkedList)
  3480. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3481. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3482. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3483. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3484. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3485. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3486. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3487. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3488. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3489. at java.lang.Thread.run(Thread.java:745)
  3490.  
  3491. Locked ownable synchronizers:
  3492. - None
  3493.  
  3494. "B.defaultRpcServer.handler=43,queue=7,port=60020" - Thread t@95
  3495. java.lang.Thread.State: WAITING
  3496. at java.lang.Object.wait(Native Method)
  3497. - waiting on <2e238133> (a java.util.LinkedList)
  3498. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3499. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3500. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3501. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3502. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3503. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3504. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3505. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3506. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3507. at java.lang.Thread.run(Thread.java:745)
  3508.  
  3509. Locked ownable synchronizers:
  3510. - None
  3511.  
  3512. "B.defaultRpcServer.handler=42,queue=6,port=60020" - Thread t@94
  3513. java.lang.Thread.State: TIMED_WAITING
  3514. at java.lang.Object.wait(Native Method)
  3515. - waiting on <5d0b5a5f> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3516. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3517. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3518. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3519. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3520. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3521. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  3522. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  3523. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  3524. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  3525. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  3526. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  3527. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  3528. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3529. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3530. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3531. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3532. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3533. at java.lang.Thread.run(Thread.java:745)
  3534.  
  3535. Locked ownable synchronizers:
  3536. - None
  3537.  
  3538. "B.defaultRpcServer.handler=41,queue=5,port=60020" - Thread t@93
  3539. java.lang.Thread.State: TIMED_WAITING
  3540. at java.lang.Object.wait(Native Method)
  3541. - waiting on <3e74a67> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3542. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3543. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3544. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3545. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3546. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3547. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3548. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3549. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3550. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3551. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3552. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3553. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3554. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3555. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3556. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3557. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3558. at java.lang.Thread.run(Thread.java:745)
  3559.  
  3560. Locked ownable synchronizers:
  3561. - None
  3562.  
  3563. "B.defaultRpcServer.handler=40,queue=4,port=60020" - Thread t@92
  3564. java.lang.Thread.State: WAITING
  3565. at java.lang.Object.wait(Native Method)
  3566. - waiting on <526870c2> (a java.util.LinkedList)
  3567. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3568. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3569. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3570. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3571. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3572. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3573. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3574. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3575. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3576. at java.lang.Thread.run(Thread.java:745)
  3577.  
  3578. Locked ownable synchronizers:
  3579. - None
  3580.  
  3581. "B.defaultRpcServer.handler=39,queue=3,port=60020" - Thread t@91
  3582. java.lang.Thread.State: WAITING
  3583. at java.lang.Object.wait(Native Method)
  3584. - waiting on <e96676a> (a java.util.LinkedList)
  3585. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3586. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3587. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3588. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3589. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3590. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3591. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3592. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3593. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3594. at java.lang.Thread.run(Thread.java:745)
  3595.  
  3596. Locked ownable synchronizers:
  3597. - None
  3598.  
  3599. "B.defaultRpcServer.handler=38,queue=2,port=60020" - Thread t@90
  3600. java.lang.Thread.State: TIMED_WAITING
  3601. at java.lang.Object.wait(Native Method)
  3602. - waiting on <3864c5fa> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3603. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3604. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3605. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3606. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3607. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3608. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3609. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3610. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3611. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3612. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3613. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3614. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3615. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3616. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3617. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3618. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3619. at java.lang.Thread.run(Thread.java:745)
  3620.  
  3621. Locked ownable synchronizers:
  3622. - None
  3623.  
  3624. "B.defaultRpcServer.handler=37,queue=1,port=60020" - Thread t@89
  3625. java.lang.Thread.State: WAITING
  3626. at java.lang.Object.wait(Native Method)
  3627. - waiting on <369ab690> (a java.util.LinkedList)
  3628. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3629. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3630. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3631. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3632. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3633. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3634. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3635. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3636. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3637. at java.lang.Thread.run(Thread.java:745)
  3638.  
  3639. Locked ownable synchronizers:
  3640. - None
  3641.  
  3642. "B.defaultRpcServer.handler=36,queue=0,port=60020" - Thread t@88
  3643. java.lang.Thread.State: TIMED_WAITING
  3644. at java.lang.Object.wait(Native Method)
  3645. - waiting on <3c9d17> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3646. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3647. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3648. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3649. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3650. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3651. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3652. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3653. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3654. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3655. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3656. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3657. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3658. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3659. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3660. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3661. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3662. at java.lang.Thread.run(Thread.java:745)
  3663.  
  3664. Locked ownable synchronizers:
  3665. - None
  3666.  
  3667. "B.defaultRpcServer.handler=35,queue=11,port=60020" - Thread t@87
  3668. java.lang.Thread.State: WAITING
  3669. at java.lang.Object.wait(Native Method)
  3670. - waiting on <4bd2726f> (a java.util.LinkedList)
  3671. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3672. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3673. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3674. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3675. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3676. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3677. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3678. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3679. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3680. at java.lang.Thread.run(Thread.java:745)
  3681.  
  3682. Locked ownable synchronizers:
  3683. - None
  3684.  
  3685. "B.defaultRpcServer.handler=34,queue=10,port=60020" - Thread t@86
  3686. java.lang.Thread.State: TIMED_WAITING
  3687. at java.lang.Object.wait(Native Method)
  3688. - waiting on <10270bd1> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3689. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3690. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3691. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3692. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3693. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3694. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3695. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3696. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3697. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3698. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3699. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3700. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3701. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3702. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3703. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3704. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3705. at java.lang.Thread.run(Thread.java:745)
  3706.  
  3707. Locked ownable synchronizers:
  3708. - None
  3709.  
  3710. "B.defaultRpcServer.handler=33,queue=9,port=60020" - Thread t@85
  3711. java.lang.Thread.State: WAITING
  3712. at java.lang.Object.wait(Native Method)
  3713. - waiting on <369ab690> (a java.util.LinkedList)
  3714. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3715. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3716. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3717. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3718. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3719. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3720. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3721. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3722. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3723. at java.lang.Thread.run(Thread.java:745)
  3724.  
  3725. Locked ownable synchronizers:
  3726. - None
  3727.  
  3728. "B.defaultRpcServer.handler=32,queue=8,port=60020" - Thread t@84
  3729. java.lang.Thread.State: WAITING
  3730. at java.lang.Object.wait(Native Method)
  3731. - waiting on <4514ce22> (a java.util.LinkedList)
  3732. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3733. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3734. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3735. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3736. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3737. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3738. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3739. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3740. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3741. at java.lang.Thread.run(Thread.java:745)
  3742.  
  3743. Locked ownable synchronizers:
  3744. - None
  3745.  
  3746. "B.defaultRpcServer.handler=31,queue=7,port=60020" - Thread t@83
  3747. java.lang.Thread.State: WAITING
  3748. at java.lang.Object.wait(Native Method)
  3749. - waiting on <4514ce22> (a java.util.LinkedList)
  3750. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3751. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3752. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3753. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3754. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3755. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3756. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3757. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3758. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3759. at java.lang.Thread.run(Thread.java:745)
  3760.  
  3761. Locked ownable synchronizers:
  3762. - None
  3763.  
  3764. "B.defaultRpcServer.handler=30,queue=6,port=60020" - Thread t@82
  3765. java.lang.Thread.State: TIMED_WAITING
  3766. at java.lang.Object.wait(Native Method)
  3767. - waiting on <219881c0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3768. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3769. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3770. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3771. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3772. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3773. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3774. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3775. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3776. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3777. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3778. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3779. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3780. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3781. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3782. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3783. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3784. at java.lang.Thread.run(Thread.java:745)
  3785.  
  3786. Locked ownable synchronizers:
  3787. - None
  3788.  
  3789. "B.defaultRpcServer.handler=29,queue=5,port=60020" - Thread t@81
  3790. java.lang.Thread.State: TIMED_WAITING
  3791. at java.lang.Object.wait(Native Method)
  3792. - waiting on <6042e48d> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3793. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3794. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3795. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3796. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3797. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3798. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3799. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3800. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3801. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3802. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3803. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3804. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3805. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3806. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3807. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3808. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3809. at java.lang.Thread.run(Thread.java:745)
  3810.  
  3811. Locked ownable synchronizers:
  3812. - None
  3813.  
  3814. "B.defaultRpcServer.handler=28,queue=4,port=60020" - Thread t@80
  3815. java.lang.Thread.State: WAITING
  3816. at java.lang.Object.wait(Native Method)
  3817. - waiting on <526870c2> (a java.util.LinkedList)
  3818. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3819. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3820. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3821. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3822. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3823. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3824. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3825. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3826. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3827. at java.lang.Thread.run(Thread.java:745)
  3828.  
  3829. Locked ownable synchronizers:
  3830. - None
  3831.  
  3832. "B.defaultRpcServer.handler=27,queue=3,port=60020" - Thread t@79
  3833. java.lang.Thread.State: WAITING
  3834. at java.lang.Object.wait(Native Method)
  3835. - waiting on <e96676a> (a java.util.LinkedList)
  3836. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3837. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3838. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3839. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3840. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3841. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3842. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3843. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3844. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3845. at java.lang.Thread.run(Thread.java:745)
  3846.  
  3847. Locked ownable synchronizers:
  3848. - None
  3849.  
  3850. "B.defaultRpcServer.handler=26,queue=2,port=60020" - Thread t@78
  3851. java.lang.Thread.State: TIMED_WAITING
  3852. at java.lang.Object.wait(Native Method)
  3853. - waiting on <415b6b56> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3854. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3855. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3856. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3857. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3858. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3859. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  3860. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  3861. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  3862. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  3863. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  3864. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  3865. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  3866. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3867. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3868. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3869. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3870. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3871. at java.lang.Thread.run(Thread.java:745)
  3872.  
  3873. Locked ownable synchronizers:
  3874. - None
  3875.  
  3876. "B.defaultRpcServer.handler=25,queue=1,port=60020" - Thread t@77
  3877. java.lang.Thread.State: TIMED_WAITING
  3878. at java.lang.Object.wait(Native Method)
  3879. - waiting on <281267b5> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3880. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3881. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3882. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3883. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3884. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3885. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3886. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3887. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3888. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3889. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3890. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3891. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3892. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3893. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3894. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3895. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3896. at java.lang.Thread.run(Thread.java:745)
  3897.  
  3898. Locked ownable synchronizers:
  3899. - None
  3900.  
  3901. "B.defaultRpcServer.handler=24,queue=0,port=60020" - Thread t@76
  3902. java.lang.Thread.State: TIMED_WAITING
  3903. at java.lang.Object.wait(Native Method)
  3904. - waiting on <717d11a0> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3905. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3906. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3907. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3908. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3909. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3910. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3911. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3912. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3913. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3914. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3915. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3916. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3917. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3918. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3919. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3920. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3921. at java.lang.Thread.run(Thread.java:745)
  3922.  
  3923. Locked ownable synchronizers:
  3924. - None
  3925.  
  3926. "B.defaultRpcServer.handler=23,queue=11,port=60020" - Thread t@75
  3927. java.lang.Thread.State: WAITING
  3928. at java.lang.Object.wait(Native Method)
  3929. - waiting on <7205131a> (a java.util.LinkedList)
  3930. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3931. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3932. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3933. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3934. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3935. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3936. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3937. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3938. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3939. at java.lang.Thread.run(Thread.java:745)
  3940.  
  3941. Locked ownable synchronizers:
  3942. - None
  3943.  
  3944. "B.defaultRpcServer.handler=22,queue=10,port=60020" - Thread t@74
  3945. java.lang.Thread.State: WAITING
  3946. at java.lang.Object.wait(Native Method)
  3947. - waiting on <e96676a> (a java.util.LinkedList)
  3948. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  3949. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  3950. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  3951. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  3952. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  3953. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3954. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3955. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3956. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3957. at java.lang.Thread.run(Thread.java:745)
  3958.  
  3959. Locked ownable synchronizers:
  3960. - None
  3961.  
  3962. "B.defaultRpcServer.handler=21,queue=9,port=60020" - Thread t@73
  3963. java.lang.Thread.State: TIMED_WAITING
  3964. at java.lang.Object.wait(Native Method)
  3965. - waiting on <4255a4a9> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3966. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3967. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3968. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3969. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3970. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3971. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  3972. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  3973. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  3974. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  3975. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  3976. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  3977. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  3978. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  3979. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  3980. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  3981. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  3982. at java.lang.Thread.run(Thread.java:745)
  3983.  
  3984. Locked ownable synchronizers:
  3985. - None
  3986.  
  3987. "B.defaultRpcServer.handler=20,queue=8,port=60020" - Thread t@72
  3988. java.lang.Thread.State: TIMED_WAITING
  3989. at java.lang.Object.wait(Native Method)
  3990. - waiting on <4754e530> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  3991. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  3992. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  3993. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  3994. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  3995. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  3996. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  3997. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  3998. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  3999. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  4000. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  4001. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  4002. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  4003. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4004. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4005. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4006. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4007. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4008. at java.lang.Thread.run(Thread.java:745)
  4009.  
  4010. Locked ownable synchronizers:
  4011. - None
  4012.  
  4013. "B.defaultRpcServer.handler=19,queue=7,port=60020" - Thread t@71
  4014. java.lang.Thread.State: WAITING
  4015. at java.lang.Object.wait(Native Method)
  4016. - waiting on <78afd1e> (a java.util.LinkedList)
  4017. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  4018. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  4019. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  4020. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  4021. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  4022. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4023. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4024. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4025. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4026. at java.lang.Thread.run(Thread.java:745)
  4027.  
  4028. Locked ownable synchronizers:
  4029. - None
  4030.  
  4031. "B.defaultRpcServer.handler=18,queue=6,port=60020" - Thread t@70
  4032. java.lang.Thread.State: TIMED_WAITING
  4033. at java.lang.Object.wait(Native Method)
  4034. - waiting on <3b66ef3d> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4035. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4036. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4037. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4038. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4039. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4040. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4041. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4042. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4043. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4044. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4045. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4046. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4047. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4048. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4049. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4050. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4051. at java.lang.Thread.run(Thread.java:745)
  4052.  
  4053. Locked ownable synchronizers:
  4054. - None
  4055.  
  4056. "B.defaultRpcServer.handler=17,queue=5,port=60020" - Thread t@69
  4057. java.lang.Thread.State: TIMED_WAITING
  4058. at java.lang.Object.wait(Native Method)
  4059. - waiting on <17e682c9> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4060. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4061. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4062. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4063. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4064. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4065. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4066. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4067. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4068. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4069. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4070. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4071. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4072. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4073. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4074. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4075. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4076. at java.lang.Thread.run(Thread.java:745)
  4077.  
  4078. Locked ownable synchronizers:
  4079. - None
  4080.  
  4081. "B.defaultRpcServer.handler=16,queue=4,port=60020" - Thread t@68
  4082. java.lang.Thread.State: WAITING
  4083. at java.lang.Object.wait(Native Method)
  4084. - waiting on <4514ce22> (a java.util.LinkedList)
  4085. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  4086. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  4087. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  4088. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  4089. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  4090. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4091. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4092. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4093. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4094. at java.lang.Thread.run(Thread.java:745)
  4095.  
  4096. Locked ownable synchronizers:
  4097. - None
  4098.  
  4099. "B.defaultRpcServer.handler=15,queue=3,port=60020" - Thread t@67
  4100. java.lang.Thread.State: TIMED_WAITING
  4101. at java.lang.Object.wait(Native Method)
  4102. - waiting on <2e4a3f0b> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4103. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4104. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4105. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4106. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4107. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4108. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4109. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4110. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4111. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4112. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4113. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4114. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4115. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4116. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4117. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4118. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4119. at java.lang.Thread.run(Thread.java:745)
  4120.  
  4121. Locked ownable synchronizers:
  4122. - None
  4123.  
  4124. "B.defaultRpcServer.handler=14,queue=2,port=60020" - Thread t@66
  4125. java.lang.Thread.State: TIMED_WAITING
  4126. at java.lang.Object.wait(Native Method)
  4127. - waiting on <da3fafe> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4128. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4129. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4130. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4131. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4132. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4133. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4134. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4135. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4136. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4137. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4138. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4139. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4140. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4141. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4142. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4143. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4144. at java.lang.Thread.run(Thread.java:745)
  4145.  
  4146. Locked ownable synchronizers:
  4147. - None
  4148.  
  4149. "B.defaultRpcServer.handler=13,queue=1,port=60020" - Thread t@65
  4150. java.lang.Thread.State: WAITING
  4151. at java.lang.Object.wait(Native Method)
  4152. - waiting on <7ef82fe0> (a java.util.LinkedList)
  4153. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  4154. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  4155. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  4156. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  4157. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  4158. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4159. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4160. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4161. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4162. at java.lang.Thread.run(Thread.java:745)
  4163.  
  4164. Locked ownable synchronizers:
  4165. - None
  4166.  
  4167. "B.defaultRpcServer.handler=12,queue=0,port=60020" - Thread t@64
  4168. java.lang.Thread.State: TIMED_WAITING
  4169. at java.lang.Object.wait(Native Method)
  4170. - waiting on <63d396f7> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4171. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4172. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4173. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4174. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4175. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4176. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4177. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4178. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4179. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4180. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4181. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4182. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4183. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4184. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4185. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4186. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4187. at java.lang.Thread.run(Thread.java:745)
  4188.  
  4189. Locked ownable synchronizers:
  4190. - None
  4191.  
  4192. "B.defaultRpcServer.handler=11,queue=11,port=60020" - Thread t@63
  4193. java.lang.Thread.State: TIMED_WAITING
  4194. at java.lang.Object.wait(Native Method)
  4195. - waiting on <79dc4cf2> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4196. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4197. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4198. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4199. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4200. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4201. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4202. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4203. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4204. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4205. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4206. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4207. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4208. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4209. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4210. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4211. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4212. at java.lang.Thread.run(Thread.java:745)
  4213.  
  4214. Locked ownable synchronizers:
  4215. - None
  4216.  
  4217. "B.defaultRpcServer.handler=10,queue=10,port=60020" - Thread t@62
  4218. java.lang.Thread.State: TIMED_WAITING
  4219. at java.lang.Object.wait(Native Method)
  4220. - waiting on <134b4231> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4221. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4222. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4223. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4224. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4225. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4226. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  4227. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  4228. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  4229. at org.apache.hadoop.hbase.regionserver.HRegion.doBatchMutate(HRegion.java:3442)
  4230. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3343)
  4231. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  4232. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  4233. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4234. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4235. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4236. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4237. at java.lang.Thread.run(Thread.java:745)
  4238.  
  4239. Locked ownable synchronizers:
  4240. - None
  4241.  
  4242. "B.defaultRpcServer.handler=9,queue=9,port=60020" - Thread t@61
  4243. java.lang.Thread.State: TIMED_WAITING
  4244. at java.lang.Object.wait(Native Method)
  4245. - waiting on <270600d5> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4246. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4247. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4248. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4249. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4250. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4251. at org.apache.hadoop.hbase.regionserver.HRegion.doMiniBatchMutation(HRegion.java:3148)
  4252. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2796)
  4253. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2731)
  4254. at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:2735)
  4255. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doBatchOp(RSRpcServices.java:677)
  4256. at org.apache.hadoop.hbase.regionserver.RSRpcServices.doNonAtomicRegionMutation(RSRpcServices.java:639)
  4257. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1931)
  4258. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4259. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4260. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4261. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4262. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4263. at java.lang.Thread.run(Thread.java:745)
  4264.  
  4265. Locked ownable synchronizers:
  4266. - None
  4267.  
  4268. "B.defaultRpcServer.handler=8,queue=8,port=60020" - Thread t@60
  4269. java.lang.Thread.State: WAITING
  4270. at java.lang.Object.wait(Native Method)
  4271. - waiting on <e6e793d> (a java.util.LinkedList)
  4272. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  4273. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  4274. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  4275. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  4276. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  4277. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4278. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4279. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4280. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4281. at java.lang.Thread.run(Thread.java:745)
  4282.  
  4283. Locked ownable synchronizers:
  4284. - None
  4285.  
  4286. "B.defaultRpcServer.handler=7,queue=7,port=60020" - Thread t@59
  4287. java.lang.Thread.State: WAITING
  4288. at java.lang.Object.wait(Native Method)
  4289. - waiting on <71fe108d> (a java.util.LinkedList)
  4290. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  4291. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  4292. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  4293. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  4294. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  4295. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4296. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4297. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4298. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4299. at java.lang.Thread.run(Thread.java:745)
  4300.  
  4301. Locked ownable synchronizers:
  4302. - None
  4303.  
  4304. "B.defaultRpcServer.handler=6,queue=6,port=60020" - Thread t@58
  4305. java.lang.Thread.State: TIMED_WAITING
  4306. at java.lang.Object.wait(Native Method)
  4307. - waiting on <56181401> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4308. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4309. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4310. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4311. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4312. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4313. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4314. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4315. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4316. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4317. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4318. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4319. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4320. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4321. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4322. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4323. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4324. at java.lang.Thread.run(Thread.java:745)
  4325.  
  4326. Locked ownable synchronizers:
  4327. - None
  4328.  
  4329. "B.defaultRpcServer.handler=5,queue=5,port=60020" - Thread t@57
  4330. java.lang.Thread.State: TIMED_WAITING
  4331. at java.lang.Object.wait(Native Method)
  4332. - waiting on <72397b9f> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4333. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4334. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4335. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4336. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4337. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4338. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4339. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4340. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4341. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4342. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4343. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4344. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4345. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4346. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4347. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4348. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4349. at java.lang.Thread.run(Thread.java:745)
  4350.  
  4351. Locked ownable synchronizers:
  4352. - None
  4353.  
  4354. "B.defaultRpcServer.handler=4,queue=4,port=60020" - Thread t@56
  4355. java.lang.Thread.State: TIMED_WAITING
  4356. at java.lang.Object.wait(Native Method)
  4357. - waiting on <7a881d01> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4358. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4359. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4360. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4361. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4362. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4363. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4364. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4365. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4366. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4367. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4368. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4369. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4370. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4371. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4372. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4373. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4374. at java.lang.Thread.run(Thread.java:745)
  4375.  
  4376. Locked ownable synchronizers:
  4377. - None
  4378.  
  4379. "B.defaultRpcServer.handler=3,queue=3,port=60020" - Thread t@55
  4380. java.lang.Thread.State: TIMED_WAITING
  4381. at java.lang.Object.wait(Native Method)
  4382. - waiting on <4c54018c> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4383. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4384. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4385. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4386. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4387. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4388. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4389. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4390. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4391. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4392. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4393. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4394. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4395. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4396. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4397. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4398. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4399. at java.lang.Thread.run(Thread.java:745)
  4400.  
  4401. Locked ownable synchronizers:
  4402. - None
  4403.  
  4404. "B.defaultRpcServer.handler=2,queue=2,port=60020" - Thread t@54
  4405. java.lang.Thread.State: TIMED_WAITING
  4406. at java.lang.Object.wait(Native Method)
  4407. - waiting on <1ff3214a> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4408. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4409. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4410. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4411. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4412. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4413. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4414. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4415. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4416. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4417. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4418. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4419. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4420. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4421. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4422. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4423. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4424. at java.lang.Thread.run(Thread.java:745)
  4425.  
  4426. Locked ownable synchronizers:
  4427. - None
  4428.  
  4429. "B.defaultRpcServer.handler=1,queue=1,port=60020" - Thread t@53
  4430. java.lang.Thread.State: WAITING
  4431. at java.lang.Object.wait(Native Method)
  4432. - waiting on <968530f> (a java.util.LinkedList)
  4433. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:242)
  4434. at org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl.waitForPreviousTransactionsComplete(MultiVersionConsistencyControl.java:221)
  4435. at org.apache.hadoop.hbase.regionserver.HRegion.checkAndMutate(HRegion.java:3287)
  4436. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2005)
  4437. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32203)
  4438. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4439. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4440. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4441. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4442. at java.lang.Thread.run(Thread.java:745)
  4443.  
  4444. Locked ownable synchronizers:
  4445. - None
  4446.  
  4447. "B.defaultRpcServer.handler=0,queue=0,port=60020" - Thread t@52
  4448. java.lang.Thread.State: TIMED_WAITING
  4449. at java.lang.Object.wait(Native Method)
  4450. - waiting on <39079287> (a org.apache.hadoop.hbase.regionserver.wal.SyncFuture)
  4451. at org.apache.hadoop.hbase.regionserver.wal.SyncFuture.get(SyncFuture.java:167)
  4452. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.blockOnSync(FSHLog.java:1504)
  4453. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.publishSyncThenBlockOnCompletion(FSHLog.java:1498)
  4454. at org.apache.hadoop.hbase.regionserver.wal.FSHLog.sync(FSHLog.java:1632)
  4455. at org.apache.hadoop.hbase.regionserver.HRegion.syncOrDefer(HRegion.java:7737)
  4456. at org.apache.hadoop.hbase.regionserver.HRegion.processRowsWithLocks(HRegion.java:6504)
  4457. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6352)
  4458. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRowsWithLocks(HRegion.java:6334)
  4459. at org.apache.hadoop.hbase.regionserver.HRegion.mutateRow(HRegion.java:6325)
  4460. at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutateRows(RSRpcServices.java:418)
  4461. at org.apache.hadoop.hbase.regionserver.RSRpcServices.multi(RSRpcServices.java:1916)
  4462. at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32213)
  4463. at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2034)
  4464. at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107)
  4465. at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130)
  4466. at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107)
  4467. at java.lang.Thread.run(Thread.java:745)
  4468.  
  4469. Locked ownable synchronizers:
  4470. - None
  4471.  
  4472. "RpcServer.listener,port=60020" - Thread t@30
  4473. java.lang.Thread.State: RUNNABLE
  4474. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  4475. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  4476. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  4477. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  4478. - locked <443c3d58> (a sun.nio.ch.Util$2)
  4479. - locked <519eb54e> (a java.util.Collections$UnmodifiableSet)
  4480. - locked <35e6840e> (a sun.nio.ch.EPollSelectorImpl)
  4481. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  4482. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
  4483. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.run(RpcServer.java:678)
  4484.  
  4485. Locked ownable synchronizers:
  4486. - None
  4487.  
  4488. "RpcServer.responder" - Thread t@42
  4489. java.lang.Thread.State: RUNNABLE
  4490. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  4491. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  4492. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  4493. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  4494. - locked <5ae8f48> (a sun.nio.ch.Util$2)
  4495. - locked <20ddb458> (a java.util.Collections$UnmodifiableSet)
  4496. - locked <44d41c8> (a sun.nio.ch.EPollSelectorImpl)
  4497. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  4498. at org.apache.hadoop.hbase.ipc.RpcServer$Responder.doRunLoop(RpcServer.java:917)
  4499. at org.apache.hadoop.hbase.ipc.RpcServer$Responder.run(RpcServer.java:864)
  4500.  
  4501. Locked ownable synchronizers:
  4502. - None
  4503.  
  4504. "main-EventThread" - Thread t@51
  4505. java.lang.Thread.State: WAITING
  4506. at sun.misc.Unsafe.park(Native Method)
  4507. - parking to wait for <f6706f> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4508. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4509. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4510. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  4511. at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
  4512.  
  4513. Locked ownable synchronizers:
  4514. - None
  4515.  
  4516. "main-SendThread(ip-10-99-180-158.aolp-prd.us-east-1.ec2.aolcloud.net:2181)" - Thread t@50
  4517. java.lang.Thread.State: RUNNABLE
  4518. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  4519. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  4520. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  4521. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  4522. - locked <50a63bb> (a sun.nio.ch.Util$2)
  4523. - locked <742f789d> (a java.util.Collections$UnmodifiableSet)
  4524. - locked <399d6d34> (a sun.nio.ch.EPollSelectorImpl)
  4525. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  4526. at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
  4527. at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
  4528.  
  4529. Locked ownable synchronizers:
  4530. - None
  4531.  
  4532. "Thread-6" - Thread t@48
  4533. java.lang.Thread.State: RUNNABLE
  4534. at org.apache.hadoop.net.unix.DomainSocketWatcher.doPoll0(Native Method)
  4535. at org.apache.hadoop.net.unix.DomainSocketWatcher.access$900(DomainSocketWatcher.java:52)
  4536. at org.apache.hadoop.net.unix.DomainSocketWatcher$2.run(DomainSocketWatcher.java:509)
  4537. at java.lang.Thread.run(Thread.java:745)
  4538.  
  4539. Locked ownable synchronizers:
  4540. - None
  4541.  
  4542. "MobFileCache #0" - Thread t@47
  4543. java.lang.Thread.State: TIMED_WAITING
  4544. at sun.misc.Unsafe.park(Native Method)
  4545. - parking to wait for <719af4b3> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4546. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  4547. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  4548. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  4549. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  4550. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  4551. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  4552. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4553. at java.lang.Thread.run(Thread.java:745)
  4554.  
  4555. Locked ownable synchronizers:
  4556. - None
  4557.  
  4558. "LruBlockCacheStatsExecutor" - Thread t@45
  4559. java.lang.Thread.State: TIMED_WAITING
  4560. at sun.misc.Unsafe.park(Native Method)
  4561. - parking to wait for <79b237a9> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4562. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  4563. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  4564. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
  4565. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
  4566. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  4567. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  4568. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4569. at java.lang.Thread.run(Thread.java:745)
  4570.  
  4571. Locked ownable synchronizers:
  4572. - None
  4573.  
  4574. "main.LruBlockCache.EvictionThread" - Thread t@43
  4575. java.lang.Thread.State: TIMED_WAITING
  4576. at java.lang.Object.wait(Native Method)
  4577. - waiting on <448c51f4> (a org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread)
  4578. at org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:791)
  4579. at java.lang.Thread.run(Thread.java:745)
  4580.  
  4581. Locked ownable synchronizers:
  4582. - None
  4583.  
  4584. "Timer for 'HBase' metrics system" - Thread t@41
  4585. java.lang.Thread.State: TIMED_WAITING
  4586. at java.lang.Object.wait(Native Method)
  4587. - waiting on <44d52de2> (a java.util.TaskQueue)
  4588. at java.util.TimerThread.mainLoop(Timer.java:552)
  4589. at java.util.TimerThread.run(Timer.java:505)
  4590.  
  4591. Locked ownable synchronizers:
  4592. - None
  4593.  
  4594. "RpcServer.reader=9,bindAddress=0.0.0.0,port=60020" - Thread t@40
  4595. java.lang.Thread.State: WAITING
  4596. at sun.misc.Unsafe.park(Native Method)
  4597. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4598. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4599. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4600. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4601. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4602. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4603. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4604. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4605. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4606. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4607. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4608. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4609. - locked <573028ce> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4610. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4611. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4612. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4613. at java.lang.Thread.run(Thread.java:745)
  4614.  
  4615. Locked ownable synchronizers:
  4616. - locked <75c56eb9> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4617.  
  4618. "RpcServer.reader=8,bindAddress=0.0.0.0,port=60020" - Thread t@39
  4619. java.lang.Thread.State: WAITING
  4620. at sun.misc.Unsafe.park(Native Method)
  4621. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4622. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4623. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4624. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4625. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4626. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4627. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4628. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4629. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4630. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4631. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4632. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4633. - locked <1719e28> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4634. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4635. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4636. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4637. at java.lang.Thread.run(Thread.java:745)
  4638.  
  4639. Locked ownable synchronizers:
  4640. - locked <57a3e26a> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4641.  
  4642. "RpcServer.reader=7,bindAddress=0.0.0.0,port=60020" - Thread t@38
  4643. java.lang.Thread.State: WAITING
  4644. at sun.misc.Unsafe.park(Native Method)
  4645. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4646. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4647. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4648. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4649. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4650. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4651. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4652. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4653. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4654. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4655. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4656. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4657. - locked <e15f50b> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4658. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4659. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4660. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4661. at java.lang.Thread.run(Thread.java:745)
  4662.  
  4663. Locked ownable synchronizers:
  4664. - locked <4b2c5e02> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4665.  
  4666. "RpcServer.reader=6,bindAddress=0.0.0.0,port=60020" - Thread t@37
  4667. java.lang.Thread.State: WAITING
  4668. at sun.misc.Unsafe.park(Native Method)
  4669. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4670. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4671. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4672. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4673. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4674. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4675. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4676. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4677. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4678. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4679. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4680. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4681. - locked <5c33cd30> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4682. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4683. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4684. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4685. at java.lang.Thread.run(Thread.java:745)
  4686.  
  4687. Locked ownable synchronizers:
  4688. - locked <a1f72f5> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4689.  
  4690. "RpcServer.reader=5,bindAddress=0.0.0.0,port=60020" - Thread t@36
  4691. java.lang.Thread.State: WAITING
  4692. at sun.misc.Unsafe.park(Native Method)
  4693. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4694. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4695. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4696. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4697. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4698. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4699. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4700. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4701. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4702. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4703. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4704. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4705. - locked <a13f820> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4706. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4707. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4708. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4709. at java.lang.Thread.run(Thread.java:745)
  4710.  
  4711. Locked ownable synchronizers:
  4712. - locked <19976a65> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4713.  
  4714. "RpcServer.reader=4,bindAddress=0.0.0.0,port=60020" - Thread t@35
  4715. java.lang.Thread.State: WAITING
  4716. at sun.misc.Unsafe.park(Native Method)
  4717. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4718. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4719. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4720. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4721. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4722. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4723. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4724. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4725. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4726. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4727. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4728. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4729. - locked <79eb794> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4730. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4731. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4732. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4733. at java.lang.Thread.run(Thread.java:745)
  4734.  
  4735. Locked ownable synchronizers:
  4736. - locked <2f666ebb> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4737.  
  4738. "RpcServer.reader=3,bindAddress=0.0.0.0,port=60020" - Thread t@34
  4739. java.lang.Thread.State: WAITING
  4740. at sun.misc.Unsafe.park(Native Method)
  4741. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4742. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4743. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4744. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4745. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4746. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4747. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4748. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4749. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4750. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4751. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4752. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4753. - locked <2682b7c1> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4754. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4755. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4756. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4757. at java.lang.Thread.run(Thread.java:745)
  4758.  
  4759. Locked ownable synchronizers:
  4760. - locked <dc9876b> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4761.  
  4762. "RpcServer.reader=2,bindAddress=0.0.0.0,port=60020" - Thread t@33
  4763. java.lang.Thread.State: WAITING
  4764. at sun.misc.Unsafe.park(Native Method)
  4765. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4766. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4767. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4768. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4769. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4770. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4771. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4772. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4773. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4774. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4775. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4776. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4777. - locked <649aa46c> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4778. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4779. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4780. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4781. at java.lang.Thread.run(Thread.java:745)
  4782.  
  4783. Locked ownable synchronizers:
  4784. - locked <471a9022> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4785.  
  4786. "RpcServer.reader=1,bindAddress=0.0.0.0,port=60020" - Thread t@32
  4787. java.lang.Thread.State: WAITING
  4788. at sun.misc.Unsafe.park(Native Method)
  4789. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4790. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4791. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4792. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4793. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4794. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4795. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4796. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4797. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4798. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4799. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4800. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4801. - locked <715d8227> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4802. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4803. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4804. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4805. at java.lang.Thread.run(Thread.java:745)
  4806.  
  4807. Locked ownable synchronizers:
  4808. - locked <15f47664> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4809.  
  4810. "RpcServer.reader=0,bindAddress=0.0.0.0,port=60020" - Thread t@31
  4811. java.lang.Thread.State: WAITING
  4812. at sun.misc.Unsafe.park(Native Method)
  4813. - parking to wait for <4ff176c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4814. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4815. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4816. at org.apache.hadoop.hbase.util.BoundedPriorityBlockingQueue.put(BoundedPriorityBlockingQueue.java:180)
  4817. at org.apache.hadoop.hbase.ipc.BalancedQueueRpcExecutor.dispatch(BalancedQueueRpcExecutor.java:77)
  4818. at org.apache.hadoop.hbase.ipc.SimpleRpcScheduler.dispatch(SimpleRpcScheduler.java:201)
  4819. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processRequest(RpcServer.java:1779)
  4820. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.processOneRpc(RpcServer.java:1681)
  4821. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.process(RpcServer.java:1548)
  4822. at org.apache.hadoop.hbase.ipc.RpcServer$Connection.readAndProcess(RpcServer.java:1528)
  4823. at org.apache.hadoop.hbase.ipc.RpcServer$Listener.doRead(RpcServer.java:801)
  4824. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.doRunLoop(RpcServer.java:586)
  4825. - locked <30db98f> (a org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader)
  4826. at org.apache.hadoop.hbase.ipc.RpcServer$Listener$Reader.run(RpcServer.java:562)
  4827. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4828. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4829. at java.lang.Thread.run(Thread.java:745)
  4830.  
  4831. Locked ownable synchronizers:
  4832. - locked <e25951c> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4833.  
  4834. "New I/O worker #1" - Thread t@28
  4835. java.lang.Thread.State: RUNNABLE
  4836. at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
  4837. at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
  4838. at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)
  4839. at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
  4840. - locked <30023a90> (a sun.nio.ch.Util$2)
  4841. - locked <10a4138> (a java.util.Collections$UnmodifiableSet)
  4842. - locked <1cdba784> (a sun.nio.ch.EPollSelectorImpl)
  4843. at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
  4844. at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.SelectorUtil.select(SelectorUtil.java:52)
  4845. at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:208)
  4846. at com.cloudera.cmf.event.shaded.org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:38)
  4847. at com.cloudera.cmf.event.shaded.org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:102)
  4848. at com.cloudera.cmf.event.shaded.org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42)
  4849. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  4850. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4851. at java.lang.Thread.run(Thread.java:745)
  4852.  
  4853. Locked ownable synchronizers:
  4854. - locked <7967e4ff> (a java.util.concurrent.ThreadPoolExecutor$Worker)
  4855.  
  4856. "AvroEventStorePublisher-NettyThread-boss_1" - Thread t@27
  4857. java.lang.Thread.State: WAITING
  4858. at sun.misc.Unsafe.park(Native Method)
  4859. - parking to wait for <95e33cc> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4860. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4861. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4862. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  4863. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  4864. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  4865. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4866. at java.lang.Thread.run(Thread.java:745)
  4867.  
  4868. Locked ownable synchronizers:
  4869. - None
  4870.  
  4871. "EventStorePublisherWithRetry-0" - Thread t@24
  4872. java.lang.Thread.State: WAITING
  4873. at sun.misc.Unsafe.park(Native Method)
  4874. - parking to wait for <1e44b638> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4875. at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
  4876. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
  4877. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
  4878. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
  4879. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
  4880. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  4881. at java.lang.Thread.run(Thread.java:745)
  4882.  
  4883. Locked ownable synchronizers:
  4884. - None
  4885.  
  4886. "event filterer" - Thread t@23
  4887. java.lang.Thread.State: TIMED_WAITING
  4888. at sun.misc.Unsafe.park(Native Method)
  4889. - parking to wait for <7393222f> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  4890. at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
  4891. at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  4892. at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467)
  4893. at com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.runFiltering(LogEventProcessor.java:132)
  4894. at com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor.access$000(LogEventProcessor.java:28)
  4895. at com.cloudera.cmf.eventcatcher.client.logs.LogEventProcessor$1.run(LogEventProcessor.java:81)
  4896.  
  4897. Locked ownable synchronizers:
  4898. - None
  4899.  
  4900. "RMI TCP Accept-0" - Thread t@22
  4901. java.lang.Thread.State: RUNNABLE
  4902. at java.net.PlainSocketImpl.socketAccept(Native Method)
  4903. at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
  4904. at java.net.ServerSocket.implAccept(ServerSocket.java:545)
  4905. at java.net.ServerSocket.accept(ServerSocket.java:513)
  4906. at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52)
  4907. at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
  4908. at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
  4909. at java.lang.Thread.run(Thread.java:745)
  4910.  
  4911. Locked ownable synchronizers:
  4912. - None
  4913.  
  4914. "RMI TCP Accept-10005" - Thread t@21
  4915. java.lang.Thread.State: RUNNABLE
  4916. at java.net.PlainSocketImpl.socketAccept(Native Method)
  4917. at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
  4918. at java.net.ServerSocket.implAccept(ServerSocket.java:545)
  4919. at java.net.ServerSocket.accept(ServerSocket.java:513)
  4920. at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
  4921. at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
  4922. at java.lang.Thread.run(Thread.java:745)
  4923.  
  4924. Locked ownable synchronizers:
  4925. - None
  4926.  
  4927. "RMI TCP Accept-0" - Thread t@20
  4928. java.lang.Thread.State: RUNNABLE
  4929. at java.net.PlainSocketImpl.socketAccept(Native Method)
  4930. at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
  4931. at java.net.ServerSocket.implAccept(ServerSocket.java:545)
  4932. at java.net.ServerSocket.accept(ServerSocket.java:513)
  4933. at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
  4934. at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
  4935. at java.lang.Thread.run(Thread.java:745)
  4936.  
  4937. Locked ownable synchronizers:
  4938. - None
  4939.  
  4940. "Signal Dispatcher" - Thread t@5
  4941. java.lang.Thread.State: RUNNABLE
  4942.  
  4943. Locked ownable synchronizers:
  4944. - None
  4945.  
  4946. "Finalizer" - Thread t@3
  4947. java.lang.Thread.State: WAITING
  4948. at java.lang.Object.wait(Native Method)
  4949. - waiting on <babafc2> (a java.lang.ref.ReferenceQueue$Lock)
  4950. at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
  4951. at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
  4952. at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
  4953.  
  4954. Locked ownable synchronizers:
  4955. - None
  4956.  
  4957. "Reference Handler" - Thread t@2
  4958. java.lang.Thread.State: WAITING
  4959. at java.lang.Object.wait(Native Method)
  4960. - waiting on <31add175> (a java.lang.ref.Reference$Lock)
  4961. at java.lang.Object.wait(Object.java:502)
  4962. at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:157)
  4963.  
  4964. Locked ownable synchronizers:
  4965. - None
  4966.  
  4967. "main" - Thread t@1
  4968. java.lang.Thread.State: WAITING
  4969. at java.lang.Object.wait(Native Method)
  4970. - waiting on <774a2402> (a java.lang.Thread)
  4971. at java.lang.Thread.join(Thread.java:1245)
  4972. at java.lang.Thread.join(Thread.java:1319)
  4973. at org.apache.hadoop.hbase.util.HasThread.join(HasThread.java:89)
  4974. at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:66)
  4975. at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:87)
  4976. at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
  4977. at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
  4978. at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2532)
  4979.  
  4980. Locked ownable synchronizers:
  4981. - None
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!