API tools faq
paste
Paul_Zaltsman

Untitled

Paul_Zaltsman
Nov 29th, 2017
427
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 78.40 KB | None | 0 0
raw download clone embed print report
  1. [2017-11-29T06:44:13,824][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [prod-es-dn-1] failed to execute on node [1bZdHAU4TAS_QGW3_fzzzQ]
  2. org.elasticsearch.transport.RemoteTransportException: [Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]]
  3. Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]
  4. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1431) [elasticsearch-5.5.2.jar:5.5.2]
  5. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  6. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.5.2.jar:5.5.2]
  7. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  8. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  9. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  10. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  11. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  12. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  13. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  14. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  15. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  16. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  17. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  18. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  19. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  20. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  21. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  22. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  23. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  24. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  25. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  26. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  27. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  28. Caused by: java.lang.IllegalStateException: unexpected byte [0x03]
  29. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:409) ~[elasticsearch-5.5.2.jar:5.5.2]
  30. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:399) ~[elasticsearch-5.5.2.jar:5.5.2]
  31. at org.elasticsearch.common.io.stream.StreamInput.readOptionalWriteable(StreamInput.java:710) ~[elasticsearch-5.5.2.jar:5.5.2]
  32. at org.elasticsearch.action.admin.cluster.node.info.NodeInfo.readFrom(NodeInfo.java:208) ~[elasticsearch-5.5.2.jar:5.5.2]
  33. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1428) ~[elasticsearch-5.5.2.jar:5.5.2]
  34. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  35. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[?:?]
  36. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  37. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  38. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  39. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[?:?]
  40. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) ~[?:?]
  41. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) ~[?:?]
  42. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[?:?]
  43. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  44. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  45. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  46. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) ~[?:?]
  47. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  48. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  49. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) ~[?:?]
  50. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) ~[?:?]
  51. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) ~[?:?]
  52. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) ~[?:?]
  53. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) ~[?:?]
  54. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) ~[?:?]
  55. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) ~[?:?]
  56. at java.lang.Thread.run(Unknown Source) ~[?:1.8.0_151]
  57. [2017-11-29T06:44:13,843][WARN ][o.e.t.n.Netty4Transport ] [prod-es-dn-1] exception caught on transport layer [[id: 0x420136cb, L:/10.241.0.53:49178 - R:10.241.0.54/10.241.0.54:9300]], closing connection
  58. java.lang.IllegalStateException: Message not fully read (response) for requestId [2960822], handler [org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler/org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1@205e0931], error [false]; resetting
  59. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1410) ~[elasticsearch-5.5.2.jar:5.5.2]
  60. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[transport-netty4-5.5.2.jar:5.5.2]
  61. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  62. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  63. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  64. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  65. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  66. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  67. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  68. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  69. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  70. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  71. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  72. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  73. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  74. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  75. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  76. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  77. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  78. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  79. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  80. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  81. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  82. [2017-11-29T06:44:13,828][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [prod-es-dn-1] failed to execute on node [UI-sQFQpQ5id-XtE9MrWBA]
  83. org.elasticsearch.transport.RemoteTransportException: [Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]]
  84. Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]
  85. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1431) [elasticsearch-5.5.2.jar:5.5.2]
  86. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  87. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.5.2.jar:5.5.2]
  88. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  89. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  90. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  91. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  92. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  93. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  94. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  95. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  96. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  97. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  98. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  99. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  100. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  101. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  102. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  103. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  104. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  105. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  106. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  107. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  108. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  109. Caused by: java.lang.IllegalStateException: unexpected byte [0x03]
  110. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:409) ~[elasticsearch-5.5.2.jar:5.5.2]
  111. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:399) ~[elasticsearch-5.5.2.jar:5.5.2]
  112. at org.elasticsearch.common.io.stream.StreamInput.readOptionalWriteable(StreamInput.java:710) ~[elasticsearch-5.5.2.jar:5.5.2]
  113. at org.elasticsearch.action.admin.cluster.node.info.NodeInfo.readFrom(NodeInfo.java:208) ~[elasticsearch-5.5.2.jar:5.5.2]
  114. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1428) ~[elasticsearch-5.5.2.jar:5.5.2]
  115. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  116. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[?:?]
  117. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  118. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  119. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  120. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[?:?]
  121. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) ~[?:?]
  122. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) ~[?:?]
  123. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[?:?]
  124. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  125. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  126. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  127. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) ~[?:?]
  128. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  129. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  130. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) ~[?:?]
  131. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) ~[?:?]
  132. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) ~[?:?]
  133. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) ~[?:?]
  134. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) ~[?:?]
  135. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) ~[?:?]
  136. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) ~[?:?]
  137. at java.lang.Thread.run(Unknown Source) ~[?:1.8.0_151]
  138. [2017-11-29T06:44:13,844][WARN ][o.e.t.n.Netty4Transport ] [prod-es-dn-1] exception caught on transport layer [[id: 0x8de947df, L:/10.241.0.53:51026 - R:10.241.0.56/10.241.0.56:9300]], closing connection
  139. java.lang.IllegalStateException: Message not fully read (response) for requestId [2960817], handler [org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler/org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1@31a3b58b], error [false]; resetting
  140. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1410) ~[elasticsearch-5.5.2.jar:5.5.2]
  141. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[transport-netty4-5.5.2.jar:5.5.2]
  142. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  143. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  144. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  145. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  146. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  147. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  148. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  149. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  150. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  151. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  152. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  153. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  154. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  155. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  156. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  157. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  158. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  159. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  160. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  161. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  162. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  163. [2017-11-29T06:44:13,835][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [prod-es-dn-1] failed to execute on node [_cmXfbyGTdKrbMtRkKEEOQ]
  164. org.elasticsearch.transport.RemoteTransportException: [Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]]
  165. Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]
  166. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1431) [elasticsearch-5.5.2.jar:5.5.2]
  167. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  168. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.5.2.jar:5.5.2]
  169. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  170. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  171. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  172. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  173. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  174. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  175. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  176. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  177. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  178. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  179. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  180. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  181. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  182. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  183. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  184. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  185. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  186. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  187. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  188. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  189. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  190. Caused by: java.lang.IllegalStateException: unexpected byte [0x03]
  191. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:409) ~[elasticsearch-5.5.2.jar:5.5.2]
  192. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:399) ~[elasticsearch-5.5.2.jar:5.5.2]
  193. at org.elasticsearch.common.io.stream.StreamInput.readOptionalWriteable(StreamInput.java:710) ~[elasticsearch-5.5.2.jar:5.5.2]
  194. at org.elasticsearch.action.admin.cluster.node.info.NodeInfo.readFrom(NodeInfo.java:208) ~[elasticsearch-5.5.2.jar:5.5.2]
  195. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1428) ~[elasticsearch-5.5.2.jar:5.5.2]
  196. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  197. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[?:?]
  198. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  199. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  200. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  201. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[?:?]
  202. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) ~[?:?]
  203. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) ~[?:?]
  204. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[?:?]
  205. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  206. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  207. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  208. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) ~[?:?]
  209. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  210. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  211. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) ~[?:?]
  212. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) ~[?:?]
  213. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) ~[?:?]
  214. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) ~[?:?]
  215. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) ~[?:?]
  216. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) ~[?:?]
  217. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) ~[?:?]
  218. at java.lang.Thread.run(Unknown Source) ~[?:1.8.0_151]
  219. [2017-11-29T06:44:13,847][WARN ][o.e.t.n.Netty4Transport ] [prod-es-dn-1] exception caught on transport layer [[id: 0xe3a0b72b, L:/10.241.0.53:50904 - R:10.241.0.65/10.241.0.65:9300]], closing connection
  220. java.lang.IllegalStateException: Message not fully read (response) for requestId [2960818], handler [org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler/org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1@5c66ff47], error [false]; resetting
  221. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1410) ~[elasticsearch-5.5.2.jar:5.5.2]
  222. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[transport-netty4-5.5.2.jar:5.5.2]
  223. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  224. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  225. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  226. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  227. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  228. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  229. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  230. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  231. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  232. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  233. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  234. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  235. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  236. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  237. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  238. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  239. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  240. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  241. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  242. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  243. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  244. [2017-11-29T06:44:13,813][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [prod-es-dn-1] failed to execute on node [rFIchWqXT3yzLj-PXWU6qg]
  245. org.elasticsearch.transport.RemoteTransportException: [Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]]
  246. Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]
  247. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1431) [elasticsearch-5.5.2.jar:5.5.2]
  248. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  249. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.5.2.jar:5.5.2]
  250. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  251. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  252. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  253. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  254. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  255. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  256. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  257. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  258. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  259. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  260. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  261. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  262. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  263. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  264. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  265. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  266. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  267. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  268. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  269. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  270. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  271. Caused by: java.lang.IllegalStateException: unexpected byte [0x03]
  272. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:409) ~[elasticsearch-5.5.2.jar:5.5.2]
  273. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:399) ~[elasticsearch-5.5.2.jar:5.5.2]
  274. at org.elasticsearch.common.io.stream.StreamInput.readOptionalWriteable(StreamInput.java:710) ~[elasticsearch-5.5.2.jar:5.5.2]
  275. at org.elasticsearch.action.admin.cluster.node.info.NodeInfo.readFrom(NodeInfo.java:208) ~[elasticsearch-5.5.2.jar:5.5.2]
  276. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1428) ~[elasticsearch-5.5.2.jar:5.5.2]
  277. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  278. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[?:?]
  279. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  280. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  281. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  282. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[?:?]
  283. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) ~[?:?]
  284. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) ~[?:?]
  285. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[?:?]
  286. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  287. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  288. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  289. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) ~[?:?]
  290. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  291. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  292. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) ~[?:?]
  293. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) ~[?:?]
  294. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) ~[?:?]
  295. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) ~[?:?]
  296. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) ~[?:?]
  297. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) ~[?:?]
  298. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) ~[?:?]
  299. at java.lang.Thread.run(Unknown Source) ~[?:1.8.0_151]
  300. [2017-11-29T06:44:13,847][INFO ][o.e.d.z.ZenDiscovery ] [prod-es-dn-1] master_left [{prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}], reason [transport disconnected]
  301. [2017-11-29T06:44:13,848][WARN ][o.e.d.z.ZenDiscovery ] [prod-es-dn-1] master left (reason = transport disconnected), current nodes: nodes:
  302. {prod-es-dn-5}{rFIchWqXT3yzLj-PXWU6qg}{q6MXJL4vRwCIjaGmCDLg4g}{10.241.0.57}{10.241.0.57:9300}
  303. {prod-es-dn-9}{9UW6J7WvQQyTrNxmdjdB-w}{dR3uVNSxQeqOvUkY2lY6Bg}{10.241.0.67}{10.241.0.67:9300}
  304. {prod-es-dn-8}{eQxyE6CdRt2LzLGRLfg5Cg}{oG9bMgiqSPyvC56XCmfulQ}{10.241.0.66}{10.241.0.66:9300}
  305. {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}, master
  306. {prod-es-dn-3}{4tgDtJW3TmakSFTf_OAdUQ}{xhoCNNl_QdmqOk5YSsmj9Q}{10.241.0.55}{10.241.0.55:9300}
  307. {prod-es-dn-4}{UI-sQFQpQ5id-XtE9MrWBA}{1dM4WgAEQ56hQHuebewJkw}{10.241.0.56}{10.241.0.56:9300}
  308. {prod-es-dn-7}{_cmXfbyGTdKrbMtRkKEEOQ}{IiBpS9nESxKPebk_Zz9cCw}{10.241.0.65}{10.241.0.65:9300}
  309. {prod-es-ma-2}{hgyPXJdhTTW24hDx5P7Tgw}{daHBuA-LTCylAmZBRr0ZJQ}{10.241.0.60}{10.241.0.60:9300}
  310. {prod-es-ma-3}{vNydneeLTu6LwkLYbtP66A}{Ikx0H-HRTN-HCY-KaDD68g}{10.241.0.62}{10.241.0.62:9300}
  311. {prod-es-dn-10}{XhZNi3AkQg2Y8C1jiXl6Bg}{UBuQpB0XRWKH79ModCo-WQ}{10.241.0.68}{10.241.0.68:9300}
  312. {prod-es-dn-2}{1bZdHAU4TAS_QGW3_fzzzQ}{YI7lfsHfQv2DG9_DQYWLwA}{10.241.0.54}{10.241.0.54:9300}
  313. {prod-es-dn-6}{xz8xrnnkQzyP9opoWNxhOg}{Zb1gEqDpQHOVIZapEymvYw}{10.241.0.64}{10.241.0.64:9300}
  314. {prod-es-dn-1}{cyJgA2m2QWaUDzsAamM_8g}{jF4KW0TIQe-o7PYzx9k76A}{10.241.0.53}{10.241.0.53:9300}, local
  315.  
  316. [2017-11-29T06:44:13,847][WARN ][o.e.t.n.Netty4Transport ] [prod-es-dn-1] exception caught on transport layer [[id: 0x9e4c8a47, L:/10.241.0.53:47942 - R:10.241.0.57/10.241.0.57:9300]], closing connection
  317. java.lang.IllegalStateException: Message not fully read (response) for requestId [2960812], handler [org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler/org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1@36d0f383], error [false]; resetting
  318. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1410) ~[elasticsearch-5.5.2.jar:5.5.2]
  319. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[transport-netty4-5.5.2.jar:5.5.2]
  320. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  321. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  322. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  323. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  324. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  325. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  326. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  327. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  328. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  329. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  330. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  331. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  332. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  333. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  334. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  335. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  336. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  337. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  338. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  339. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  340. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  341. [2017-11-29T06:44:13,837][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [prod-es-dn-1] failed to execute on node [XhZNi3AkQg2Y8C1jiXl6Bg]
  342. org.elasticsearch.transport.RemoteTransportException: [Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]]
  343. Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]
  344. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1431) [elasticsearch-5.5.2.jar:5.5.2]
  345. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  346. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.5.2.jar:5.5.2]
  347. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  348. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  349. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  350. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  351. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  352. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  353. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  354. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  355. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  356. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  357. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  358. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  359. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  360. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  361. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  362. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  363. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  364. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  365. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  366. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  367. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  368. Caused by: java.lang.IllegalStateException: unexpected byte [0x03]
  369. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:409) ~[elasticsearch-5.5.2.jar:5.5.2]
  370. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:399) ~[elasticsearch-5.5.2.jar:5.5.2]
  371. at org.elasticsearch.common.io.stream.StreamInput.readOptionalWriteable(StreamInput.java:710) ~[elasticsearch-5.5.2.jar:5.5.2]
  372. at org.elasticsearch.action.admin.cluster.node.info.NodeInfo.readFrom(NodeInfo.java:208) ~[elasticsearch-5.5.2.jar:5.5.2]
  373. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1428) ~[elasticsearch-5.5.2.jar:5.5.2]
  374. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  375. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[?:?]
  376. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  377. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  378. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  379. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[?:?]
  380. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) ~[?:?]
  381. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) ~[?:?]
  382. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[?:?]
  383. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  384. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  385. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  386. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) ~[?:?]
  387. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  388. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  389. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) ~[?:?]
  390. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) ~[?:?]
  391. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) ~[?:?]
  392. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) ~[?:?]
  393. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) ~[?:?]
  394. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) ~[?:?]
  395. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) ~[?:?]
  396. at java.lang.Thread.run(Unknown Source) ~[?:1.8.0_151]
  397. [2017-11-29T06:44:13,848][WARN ][o.e.t.n.Netty4Transport ] [prod-es-dn-1] exception caught on transport layer [[id: 0xc56894d4, L:/10.241.0.53:41350 - R:10.241.0.68/10.241.0.68:9300]], closing connection
  398. java.lang.IllegalStateException: Message not fully read (response) for requestId [2960821], handler [org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler/org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1@fa051a5], error [false]; resetting
  399. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1410) ~[elasticsearch-5.5.2.jar:5.5.2]
  400. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[transport-netty4-5.5.2.jar:5.5.2]
  401. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  402. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  403. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  404. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  405. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  406. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  407. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  408. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  409. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  410. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  411. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  412. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  413. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  414. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  415. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  416. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  417. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  418. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  419. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  420. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  421. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  422. [2017-11-29T06:44:13,840][DEBUG][o.e.a.a.c.n.i.TransportNodesInfoAction] [prod-es-dn-1] failed to execute on node [hgyPXJdhTTW24hDx5P7Tgw]
  423. org.elasticsearch.transport.RemoteTransportException: [Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]]
  424. Caused by: org.elasticsearch.transport.TransportSerializationException: Failed to deserialize response of type [org.elasticsearch.action.admin.cluster.node.info.NodeInfo]
  425. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1431) [elasticsearch-5.5.2.jar:5.5.2]
  426. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  427. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) [transport-netty4-5.5.2.jar:5.5.2]
  428. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  429. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  430. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  431. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  432. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  433. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  434. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  435. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  436. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  437. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  438. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  439. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  440. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  441. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  442. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  443. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  444. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  445. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  446. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  447. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  448. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  449. Caused by: java.lang.IllegalStateException: unexpected byte [0x03]
  450. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:409) ~[elasticsearch-5.5.2.jar:5.5.2]
  451. at org.elasticsearch.common.io.stream.StreamInput.readBoolean(StreamInput.java:399) ~[elasticsearch-5.5.2.jar:5.5.2]
  452. at org.elasticsearch.common.io.stream.StreamInput.readOptionalWriteable(StreamInput.java:710) ~[elasticsearch-5.5.2.jar:5.5.2]
  453. at org.elasticsearch.action.admin.cluster.node.info.NodeInfo.readFrom(NodeInfo.java:208) ~[elasticsearch-5.5.2.jar:5.5.2]
  454. at org.elasticsearch.transport.TcpTransport.handleResponse(TcpTransport.java:1428) ~[elasticsearch-5.5.2.jar:5.5.2]
  455. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1403) [elasticsearch-5.5.2.jar:5.5.2]
  456. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[?:?]
  457. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  458. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  459. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  460. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[?:?]
  461. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) ~[?:?]
  462. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) ~[?:?]
  463. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[?:?]
  464. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  465. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  466. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[?:?]
  467. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) ~[?:?]
  468. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) ~[?:?]
  469. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) ~[?:?]
  470. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) ~[?:?]
  471. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) ~[?:?]
  472. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) ~[?:?]
  473. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) ~[?:?]
  474. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) ~[?:?]
  475. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) ~[?:?]
  476. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) ~[?:?]
  477. at java.lang.Thread.run(Unknown Source) ~[?:1.8.0_151]
  478. [2017-11-29T06:44:13,851][WARN ][o.e.t.n.Netty4Transport ] [prod-es-dn-1] exception caught on transport layer [[id: 0x41b1ea31, L:/10.241.0.53:43938 - R:10.241.0.60/10.241.0.60:9300]], closing connection
  479. java.lang.IllegalStateException: Message not fully read (response) for requestId [2960819], handler [org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler/org.elasticsearch.action.support.nodes.TransportNodesAction$AsyncAction$1@5a1f6aae], error [false]; resetting
  480. at org.elasticsearch.transport.TcpTransport.messageReceived(TcpTransport.java:1410) ~[elasticsearch-5.5.2.jar:5.5.2]
  481. at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:74) ~[transport-netty4-5.5.2.jar:5.5.2]
  482. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  483. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  484. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  485. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  486. at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  487. at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  488. at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) [netty-codec-4.1.11.Final.jar:4.1.11.Final]
  489. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  490. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  491. at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  492. at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  493. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  494. at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  495. at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  496. at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  497. at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  498. at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  499. at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  500. at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.11.Final.jar:4.1.11.Final]
  501. at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.11.Final.jar:4.1.11.Final]
  502. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  503. [2017-11-29T06:44:14,122][WARN ][o.e.a.b.TransportShardBulkAction] [prod-es-dn-1] [[tcrraw201711][2]] failed to perform indices:data/write/bulk[s] on replica [tcrraw201711][2], node[eQxyE6CdRt2LzLGRLfg5Cg], [R], s[STARTED], a[id=uQBQvqLvQNiw15X6hkJ7OQ]
  504. org.elasticsearch.transport.NodeNotConnectedException: [prod-es-dn-8][10.241.0.66:9300] Node not connected
  505. at org.elasticsearch.transport.TcpTransport.getConnection(TcpTransport.java:654) ~[elasticsearch-5.5.2.jar:5.5.2]
  506. at org.elasticsearch.transport.TcpTransport.getConnection(TcpTransport.java:116) ~[elasticsearch-5.5.2.jar:5.5.2]
  507. at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:530) ~[elasticsearch-5.5.2.jar:5.5.2]
  508. at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:506) [elasticsearch-5.5.2.jar:5.5.2]
  509. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicasProxy.performOn(TransportReplicationAction.java:962) [elasticsearch-5.5.2.jar:5.5.2]
  510. at org.elasticsearch.action.support.replication.ReplicationOperation.performOnReplica(ReplicationOperation.java:186) [elasticsearch-5.5.2.jar:5.5.2]
  511. at org.elasticsearch.action.support.replication.ReplicationOperation.performOnReplicas(ReplicationOperation.java:170) [elasticsearch-5.5.2.jar:5.5.2]
  512. at org.elasticsearch.action.support.replication.ReplicationOperation.execute(ReplicationOperation.java:130) [elasticsearch-5.5.2.jar:5.5.2]
  513. at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:322) [elasticsearch-5.5.2.jar:5.5.2]
  514. at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.java:264) [elasticsearch-5.5.2.jar:5.5.2]
  515. at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:888) [elasticsearch-5.5.2.jar:5.5.2]
  516. at org.elasticsearch.action.support.replication.TransportReplicationAction$1.onResponse(TransportReplicationAction.java:885) [elasticsearch-5.5.2.jar:5.5.2]
  517. at org.elasticsearch.index.shard.IndexShardOperationsLock.acquire(IndexShardOperationsLock.java:151) [elasticsearch-5.5.2.jar:5.5.2]
  518. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOperationLock(IndexShard.java:1657) [elasticsearch-5.5.2.jar:5.5.2]
  519. at org.elasticsearch.action.support.replication.TransportReplicationAction.acquirePrimaryShardReference(TransportReplicationAction.java:897) [elasticsearch-5.5.2.jar:5.5.2]
  520. at org.elasticsearch.action.support.replication.TransportReplicationAction.access$400(TransportReplicationAction.java:93) [elasticsearch-5.5.2.jar:5.5.2]
  521. at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:281) [elasticsearch-5.5.2.jar:5.5.2]
  522. at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.5.2.jar:5.5.2]
  523. at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:260) [elasticsearch-5.5.2.jar:5.5.2]
  524. at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:252) [elasticsearch-5.5.2.jar:5.5.2]
  525. at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) [elasticsearch-5.5.2.jar:5.5.2]
  526. at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:644) [elasticsearch-5.5.2.jar:5.5.2]
  527. at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638) [elasticsearch-5.5.2.jar:5.5.2]
  528. at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-5.5.2.jar:5.5.2]
  529. at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:1.8.0_151]
  530. at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:1.8.0_151]
  531. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  532. [2017-11-29T06:44:14,122][WARN ][o.e.c.a.s.ShardStateAction] [prod-es-dn-1] [tcrraw201711][2] no master known for action [internal:cluster/shard/failure] for shard entry [shard id [[tcrraw201711][2]], allocation id [uQBQvqLvQNiw15X6hkJ7OQ], primary term [139], message [failed to perform indices:data/write/bulk[s] on replica [tcrraw201711][2], node[eQxyE6CdRt2LzLGRLfg5Cg], [R], s[STARTED], a[id=uQBQvqLvQNiw15X6hkJ7OQ]], failure [NodeNotConnectedException[[prod-es-dn-8][10.241.0.66:9300] Node not connected]]]
  533. [2017-11-29T06:44:16,857][INFO ][o.e.c.s.ClusterService ] [prod-es-dn-1] detected_master {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}, reason: zen-disco-receive(from master [master {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300} committed version [13744]])
  534. [2017-11-29T06:44:44,491][WARN ][o.e.a.b.TransportShardBulkAction] [prod-es-dn-1] [[linklatencyrep20171129][4]] failed to perform indices:data/write/bulk[s] on replica [linklatencyrep20171129][4], node[4tgDtJW3TmakSFTf_OAdUQ], [R], s[STARTED], a[id=63dW-_v5S1eqw1sRtCYzyg]
  535. org.elasticsearch.transport.RemoteTransportException: [prod-es-dn-3][10.241.0.55:9300][indices:data/write/bulk[s][r]]
  536. Caused by: java.lang.IllegalArgumentException: [linklatencyrep20171129][4] operation term [1] is too old (current [2])
  537. at org.elasticsearch.index.shard.IndexShard.acquireReplicaOperationLock(IndexShard.java:1669) ~[elasticsearch-5.5.2.jar:5.5.2]
  538. at org.elasticsearch.action.support.replication.TransportReplicationAction$AsyncReplicaAction.doRun(TransportReplicationAction.java:566) ~[elasticsearch-5.5.2.jar:5.5.2]
  539. at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-5.5.2.jar:5.5.2]
  540. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:451) ~[elasticsearch-5.5.2.jar:5.5.2]
  541. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReplicaOperationTransportHandler.messageReceived(TransportReplicationAction.java:441) ~[elasticsearch-5.5.2.jar:5.5.2]
  542. at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:69) ~[elasticsearch-5.5.2.jar:5.5.2]
  543. at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1544) ~[elasticsearch-5.5.2.jar:5.5.2]
  544. at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:638) ~[elasticsearch-5.5.2.jar:5.5.2]
  545. at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-5.5.2.jar:5.5.2]
  546. at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:1.8.0_151]
  547. at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:1.8.0_151]
  548. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  549. [2017-11-29T06:44:46,867][INFO ][o.e.d.z.ZenDiscovery ] [prod-es-dn-1] master_left [{prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}], reason [no longer master]
  550. org.elasticsearch.transport.RemoteTransportException: [prod-es-ma-1][10.241.0.61:9300][internal:discovery/zen/fd/master_ping]
  551. Caused by: org.elasticsearch.cluster.NotMasterException: local node is not master
  552. [2017-11-29T06:44:46,868][WARN ][o.e.d.z.ZenDiscovery ] [prod-es-dn-1] master left (reason = no longer master), current nodes: nodes:
  553. {prod-es-dn-5}{rFIchWqXT3yzLj-PXWU6qg}{q6MXJL4vRwCIjaGmCDLg4g}{10.241.0.57}{10.241.0.57:9300}
  554. {prod-es-dn-9}{9UW6J7WvQQyTrNxmdjdB-w}{dR3uVNSxQeqOvUkY2lY6Bg}{10.241.0.67}{10.241.0.67:9300}
  555. {prod-es-dn-8}{eQxyE6CdRt2LzLGRLfg5Cg}{oG9bMgiqSPyvC56XCmfulQ}{10.241.0.66}{10.241.0.66:9300}
  556. {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}, master
  557. {prod-es-dn-3}{4tgDtJW3TmakSFTf_OAdUQ}{xhoCNNl_QdmqOk5YSsmj9Q}{10.241.0.55}{10.241.0.55:9300}
  558. {prod-es-dn-4}{UI-sQFQpQ5id-XtE9MrWBA}{1dM4WgAEQ56hQHuebewJkw}{10.241.0.56}{10.241.0.56:9300}
  559. {prod-es-dn-7}{_cmXfbyGTdKrbMtRkKEEOQ}{IiBpS9nESxKPebk_Zz9cCw}{10.241.0.65}{10.241.0.65:9300}
  560. {prod-es-ma-2}{hgyPXJdhTTW24hDx5P7Tgw}{daHBuA-LTCylAmZBRr0ZJQ}{10.241.0.60}{10.241.0.60:9300}
  561. {prod-es-ma-3}{vNydneeLTu6LwkLYbtP66A}{Ikx0H-HRTN-HCY-KaDD68g}{10.241.0.62}{10.241.0.62:9300}
  562. {prod-es-dn-10}{XhZNi3AkQg2Y8C1jiXl6Bg}{UBuQpB0XRWKH79ModCo-WQ}{10.241.0.68}{10.241.0.68:9300}
  563. {prod-es-dn-2}{1bZdHAU4TAS_QGW3_fzzzQ}{YI7lfsHfQv2DG9_DQYWLwA}{10.241.0.54}{10.241.0.54:9300}
  564. {prod-es-dn-6}{xz8xrnnkQzyP9opoWNxhOg}{Zb1gEqDpQHOVIZapEymvYw}{10.241.0.64}{10.241.0.64:9300}
  565. {prod-es-dn-1}{cyJgA2m2QWaUDzsAamM_8g}{jF4KW0TIQe-o7PYzx9k76A}{10.241.0.53}{10.241.0.53:9300}, local
  566.  
  567. [2017-11-29T06:44:51,877][INFO ][o.e.c.s.ClusterService ] [prod-es-dn-1] detected_master {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}, reason: zen-disco-receive(from master [master {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300} committed version [13759]])
  568. [2017-11-29T06:45:05,492][WARN ][o.e.c.a.s.ShardStateAction] [prod-es-dn-1] [linklatencyrep20171129][4] unexpected failure while sending request [internal:cluster/shard/failure] to [{prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}] for shard entry [shard id [[linklatencyrep20171129][4]], allocation id [63dW-_v5S1eqw1sRtCYzyg], primary term [1], message [failed to perform indices:data/write/bulk[s] on replica [linklatencyrep20171129][4], node[4tgDtJW3TmakSFTf_OAdUQ], [R], s[STARTED], a[id=63dW-_v5S1eqw1sRtCYzyg]], failure [RemoteTransportException[[prod-es-dn-3][10.241.0.55:9300][indices:data/write/bulk[s][r]]]; nested: IllegalArgumentException[[linklatencyrep20171129][4] operation term [1] is too old (current [2])]; ]]
  569. org.elasticsearch.transport.RemoteTransportException: [prod-es-ma-1][10.241.0.61:9300][internal:cluster/shard/failure]
  570. Caused by: org.elasticsearch.cluster.action.shard.ShardStateAction$NoLongerPrimaryShardException: primary term [1] did not match current primary term [2]
  571. at org.elasticsearch.cluster.action.shard.ShardStateAction$ShardFailedClusterStateTaskExecutor.execute(ShardStateAction.java:291) ~[elasticsearch-5.5.2.jar:5.5.2]
  572. at org.elasticsearch.cluster.service.ClusterService.executeTasks(ClusterService.java:634) ~[elasticsearch-5.5.2.jar:5.5.2]
  573. at org.elasticsearch.cluster.service.ClusterService.calculateTaskOutputs(ClusterService.java:612) ~[elasticsearch-5.5.2.jar:5.5.2]
  574. at org.elasticsearch.cluster.service.ClusterService.runTasks(ClusterService.java:571) ~[elasticsearch-5.5.2.jar:5.5.2]
  575. at org.elasticsearch.cluster.service.ClusterService$ClusterServiceTaskBatcher.run(ClusterService.java:263) ~[elasticsearch-5.5.2.jar:5.5.2]
  576. at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-5.5.2.jar:5.5.2]
  577. at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) ~[elasticsearch-5.5.2.jar:5.5.2]
  578. at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569) ~[elasticsearch-5.5.2.jar:5.5.2]
  579. at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:247) ~[elasticsearch-5.5.2.jar:5.5.2]
  580. at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:210) ~[elasticsearch-5.5.2.jar:5.5.2]
  581. at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:1.8.0_151]
  582. at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:1.8.0_151]
  583. at java.lang.Thread.run(Unknown Source) [?:1.8.0_151]
  584. [2017-11-29T06:49:10,166][INFO ][o.e.c.s.ClusterService ] [prod-es-dn-1] removed {{prod-es-dn-2}{1bZdHAU4TAS_QGW3_fzzzQ}{YI7lfsHfQv2DG9_DQYWLwA}{10.241.0.54}{10.241.0.54:9300},}, reason: zen-disco-receive(from master [master {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300} committed version [13840]])
  585. [2017-11-29T06:49:15,120][INFO ][o.e.d.z.ZenDiscovery ] [prod-es-dn-1] master_left [{prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}], reason [no longer master]
  586. org.elasticsearch.transport.RemoteTransportException: [prod-es-ma-1][10.241.0.61:9300][internal:discovery/zen/fd/master_ping]
  587. Caused by: org.elasticsearch.cluster.NotMasterException: local node is not master
  588. [2017-11-29T06:49:15,121][WARN ][o.e.d.z.ZenDiscovery ] [prod-es-dn-1] master left (reason = no longer master), current nodes: nodes:
  589. {prod-es-dn-6}{xz8xrnnkQzyP9opoWNxhOg}{Zb1gEqDpQHOVIZapEymvYw}{10.241.0.64}{10.241.0.64:9300}
  590. {prod-es-dn-3}{4tgDtJW3TmakSFTf_OAdUQ}{xhoCNNl_QdmqOk5YSsmj9Q}{10.241.0.55}{10.241.0.55:9300}
  591. {prod-es-ma-3}{vNydneeLTu6LwkLYbtP66A}{Ikx0H-HRTN-HCY-KaDD68g}{10.241.0.62}{10.241.0.62:9300}
  592. {prod-es-dn-4}{UI-sQFQpQ5id-XtE9MrWBA}{1dM4WgAEQ56hQHuebewJkw}{10.241.0.56}{10.241.0.56:9300}
  593. {prod-es-dn-1}{cyJgA2m2QWaUDzsAamM_8g}{jF4KW0TIQe-o7PYzx9k76A}{10.241.0.53}{10.241.0.53:9300}, local
  594. {prod-es-dn-7}{_cmXfbyGTdKrbMtRkKEEOQ}{IiBpS9nESxKPebk_Zz9cCw}{10.241.0.65}{10.241.0.65:9300}
  595. {prod-es-dn-10}{XhZNi3AkQg2Y8C1jiXl6Bg}{UBuQpB0XRWKH79ModCo-WQ}{10.241.0.68}{10.241.0.68:9300}
  596. {prod-es-dn-5}{rFIchWqXT3yzLj-PXWU6qg}{q6MXJL4vRwCIjaGmCDLg4g}{10.241.0.57}{10.241.0.57:9300}
  597. {prod-es-dn-9}{9UW6J7WvQQyTrNxmdjdB-w}{dR3uVNSxQeqOvUkY2lY6Bg}{10.241.0.67}{10.241.0.67:9300}
  598. {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}, master
  599. {prod-es-dn-8}{eQxyE6CdRt2LzLGRLfg5Cg}{oG9bMgiqSPyvC56XCmfulQ}{10.241.0.66}{10.241.0.66:9300}
  600. {prod-es-ma-2}{hgyPXJdhTTW24hDx5P7Tgw}{daHBuA-LTCylAmZBRr0ZJQ}{10.241.0.60}{10.241.0.60:9300}
  601.  
  602. [2017-11-29T06:49:15,440][INFO ][o.e.c.s.ClusterService ] [prod-es-dn-1] detected_master {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300}, reason: zen-disco-receive(from master [master {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300} committed version [13841]])
  603. [2017-11-29T06:49:16,749][INFO ][o.e.c.s.ClusterService ] [prod-es-dn-1] added {{prod-es-dn-2}{1bZdHAU4TAS_QGW3_fzzzQ}{YI7lfsHfQv2DG9_DQYWLwA}{10.241.0.54}{10.241.0.54:9300},}, reason: zen-disco-receive(from master [master {prod-es-ma-1}{8scZVDYGTvaHA9B4EfCYCA}{lmHd3x5ySE-b-KsNTry0ww}{10.241.0.61}{10.241.0.61:9300} committed version [13844]])
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!