API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Feb 19th, 2020
256
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 49.01 KB | None | 0 0
raw download clone embed print report
  1. {"type": "server", "timestamp": "2020-02-19T18:21:25,601Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no known master node, scheduling a retry" }
  2. {"type": "server", "timestamp": "2020-02-19T18:21:25,602Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no known master node, scheduling a retry" }
  3. {"type": "server", "timestamp": "2020-02-19T18:21:33,730Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "last failed join attempt was 8.7s ago, failed to join {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976,
  4. ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=9, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, targetNode={es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}}]}",
  5. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [es02][10.0.7.10:9300][internal:cluster/coordination/join]",
  6. "Caused by: org.elasticsearch.transport.ConnectTransportException: [es01][10.0.0.87:9300] connect_exception",
  7. "at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:989) ~[elasticsearch-7.6.0.jar:7.6.0]",
  8. "at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:162) ~[elasticsearch-7.6.0.jar:7.6.0]",
  9. "at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  10. "at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
  11. "at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
  12. "at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
  13. "at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]",
  14. "at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  15. "at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]",
  16. "at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]",
  17. "at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]",
  18. "at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]",
  19. "at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]",
  20. "at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]",
  21. "at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]",
  22. "at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]",
  23. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]",
  24. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]",
  25. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  26. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  27. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  28. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  29. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  30. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  31. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  32. "Caused by: java.io.IOException: Connection refused: 10.0.0.87/10.0.0.87:9300",
  33. "Caused by: java.io.IOException: Connection refused",
  34. "at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
  35. "at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]",
  36. "at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]",
  37. "at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]",
  38. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]",
  39. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  40. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  41. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  42. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  43. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  44. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  45. "at java.lang.Thread.run(Thread.java:830) ~[?:?]"] }
  46. {"type": "server", "timestamp": "2020-02-19T18:21:33,732Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [SY7m05obQdCLexPLTWZFVQ, 55fLj3lWTTmWCqWUAk3vHg], have discovered [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, {es03}{uAqCgohVQGaEJNyR9kYwIg}{Mvlc3iK0QJ2uiV7HuJF_xg}{10.0.7.12}{10.0.7.12:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}, {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}] which is a quorum; discovery will continue using [10.0.7.9:9300, 10.0.7.11:9300] from hosts providers and [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state; node term 9, last-accepted version 0 in term 0" }
  47. {"type": "server", "timestamp": "2020-02-19T18:21:43,733Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [SY7m05obQdCLexPLTWZFVQ, 55fLj3lWTTmWCqWUAk3vHg], have discovered [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, {es03}{uAqCgohVQGaEJNyR9kYwIg}{Mvlc3iK0QJ2uiV7HuJF_xg}{10.0.7.12}{10.0.7.12:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}, {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}] which is a quorum; discovery will continue using [10.0.7.9:9300, 10.0.7.11:9300] from hosts providers and [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state; node term 9, last-accepted version 0 in term 0" }
  48. {"type": "server", "timestamp": "2020-02-19T18:21:46,000Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "failed to join {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=9, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, targetNode={es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}}]}",
  49. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [es02][10.0.7.10:9300][internal:cluster/coordination/join]",
  50. "Caused by: org.elasticsearch.transport.ConnectTransportException: [es01][10.0.0.87:9300] connect_exception",
  51. "at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:989) ~[elasticsearch-7.6.0.jar:7.6.0]",
  52. "at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:162) ~[elasticsearch-7.6.0.jar:7.6.0]",
  53. "at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  54. "at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
  55. "at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
  56. "at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
  57. "at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]",
  58. "at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  59. "at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[transport-netty4-client-7.6.0.jar:7.6.0]",
  60. "at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  61. "at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  62. "at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  63. "at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  64. "at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  65. "at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  66. "at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  67. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  68. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  69. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  70. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  71. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  72. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  73. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
  74. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
  75. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  76. "Caused by: java.io.IOException: Connection refused: 10.0.0.87/10.0.0.87:9300",
  77. "Caused by: java.io.IOException: Connection refused",
  78. "at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
  79. "at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]",
  80. "at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]",
  81. "at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  82. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  83. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  84. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  85. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  86. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  87. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  88. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  89. "at java.lang.Thread.run(Thread.java:830) ~[?:?]"] }
  90. {"type": "server", "timestamp": "2020-02-19T18:21:46,001Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "failed to join {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=9, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, targetNode={es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}}]}",
  91. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [es02][10.0.7.10:9300][internal:cluster/coordination/join]",
  92. "Caused by: org.elasticsearch.transport.ConnectTransportException: [es01][10.0.0.87:9300] connect_exception",
  93. "at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:989) ~[elasticsearch-7.6.0.jar:7.6.0]",
  94. "at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:162) ~[elasticsearch-7.6.0.jar:7.6.0]",
  95. "at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  96. "at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
  97. "at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
  98. "at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
  99. "at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]",
  100. "at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  101. "at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[transport-netty4-client-7.6.0.jar:7.6.0]",
  102. "at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  103. "at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  104. "at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  105. "at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  106. "at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  107. "at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  108. "at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  109. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  110. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  111. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  112. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  113. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  114. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  115. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
  116. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
  117. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  118. "Caused by: java.io.IOException: Connection refused: 10.0.0.87/10.0.0.87:9300",
  119. "Caused by: java.io.IOException: Connection refused",
  120. "at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
  121. "at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]",
  122. "at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]",
  123. "at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  124. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  125. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  126. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  127. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  128. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  129. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  130. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  131. "at java.lang.Thread.run(Thread.java:830) ~[?:?]"] }
  132. {"type": "server", "timestamp": "2020-02-19T18:21:53,734Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "last failed join attempt was 7.7s ago, failed to join {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976,
  133. ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=9, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, targetNode={es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}}]}",
  134. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [es02][10.0.7.10:9300][internal:cluster/coordination/join]",
  135. "Caused by: org.elasticsearch.transport.ConnectTransportException: [es01][10.0.0.87:9300] connect_exception",
  136. "at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:989) ~[elasticsearch-7.6.0.jar:7.6.0]",
  137. "at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:162) ~[elasticsearch-7.6.0.jar:7.6.0]",
  138. "at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  139. "at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
  140. "at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
  141. "at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
  142. "at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]",
  143. "at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  144. "at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]",
  145. "at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]",
  146. "at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]",
  147. "at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]",
  148. "at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]",
  149. "at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]",
  150. "at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]",
  151. "at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]",
  152. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]",
  153. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]",
  154. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  155. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  156. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  157. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  158. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  159. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  160. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  161. "Caused by: java.io.IOException: Connection refused: 10.0.0.87/10.0.0.87:9300",
  162. "Caused by: java.io.IOException: Connection refused",
  163. "at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
  164. "at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]",
  165. "at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]",
  166. "at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]",
  167. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]",
  168. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  169. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  170. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  171. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  172. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  173. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  174. "at java.lang.Thread.run(Thread.java:830) ~[?:?]"] }
  175. {"type": "server", "timestamp": "2020-02-19T18:21:53,736Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [SY7m05obQdCLexPLTWZFVQ, 55fLj3lWTTmWCqWUAk3vHg], have discovered [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, {es03}{uAqCgohVQGaEJNyR9kYwIg}{Mvlc3iK0QJ2uiV7HuJF_xg}{10.0.7.12}{10.0.7.12:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}, {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}] which is a quorum; discovery will continue using [10.0.7.9:9300, 10.0.7.11:9300] from hosts providers and [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state; node term 9, last-accepted version 0 in term 0" }
  176. {"type": "server", "timestamp": "2020-02-19T18:21:55,603Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "timed out while retrying [indices:admin/get] after failure (timeout [30s])" }
  177. {"type": "server", "timestamp": "2020-02-19T18:21:55,603Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "timed out while retrying [indices:admin/get] after failure (timeout [30s])" }
  178. {"type": "server", "timestamp": "2020-02-19T18:21:55,603Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "path: /.kibana, params: {index=.kibana}",
  179. "stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: null",
  180. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:220) [elasticsearch-7.6.0.jar:7.6.0]",
  181. "at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325) [elasticsearch-7.6.0.jar:7.6.0]",
  182. "at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:252) [elasticsearch-7.6.0.jar:7.6.0]",
  183. "at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:598) [elasticsearch-7.6.0.jar:7.6.0]",
  184. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633) [elasticsearch-7.6.0.jar:7.6.0]",
  185. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  186. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  187. "at java.lang.Thread.run(Thread.java:830) [?:?]"] }
  188. {"type": "server", "timestamp": "2020-02-19T18:21:55,603Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "path: /.kibana_task_manager, params: {index=.kibana_task_manager}",
  189. "stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: null",
  190. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:220) [elasticsearch-7.6.0.jar:7.6.0]",
  191. "at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325) [elasticsearch-7.6.0.jar:7.6.0]",
  192. "at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:252) [elasticsearch-7.6.0.jar:7.6.0]",
  193. "at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:598) [elasticsearch-7.6.0.jar:7.6.0]",
  194. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:633) [elasticsearch-7.6.0.jar:7.6.0]",
  195. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  196. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  197. "at java.lang.Thread.run(Thread.java:830) [?:?]"] }
  198. {"type": "server", "timestamp": "2020-02-19T18:21:58,107Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no known master node, scheduling a retry" }
  199. {"type": "server", "timestamp": "2020-02-19T18:21:58,108Z", "level": "DEBUG", "component": "o.e.a.s.m.TransportMasterNodeAction", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "no known master node, scheduling a retry" }
  200. {"type": "server", "timestamp": "2020-02-19T18:22:03,737Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [SY7m05obQdCLexPLTWZFVQ, 55fLj3lWTTmWCqWUAk3vHg], have discovered [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, {es03}{uAqCgohVQGaEJNyR9kYwIg}{Mvlc3iK0QJ2uiV7HuJF_xg}{10.0.7.12}{10.0.7.12:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}, {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}] which is a quorum; discovery will continue using [10.0.7.9:9300, 10.0.7.11:9300] from hosts providers and [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state; node term 9, last-accepted version 0 in term 0" }
  201. {"type": "server", "timestamp": "2020-02-19T18:22:07,018Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "failed to join {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=9, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, targetNode={es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}}]}",
  202. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [es02][10.0.7.10:9300][internal:cluster/coordination/join]",
  203. "Caused by: org.elasticsearch.transport.ConnectTransportException: [es01][10.0.0.87:9300] connect_exception",
  204. "at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:989) ~[elasticsearch-7.6.0.jar:7.6.0]",
  205. "at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:162) ~[elasticsearch-7.6.0.jar:7.6.0]",
  206. "at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  207. "at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
  208. "at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
  209. "at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
  210. "at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]",
  211. "at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  212. "at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[transport-netty4-client-7.6.0.jar:7.6.0]",
  213. "at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  214. "at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  215. "at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  216. "at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  217. "at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  218. "at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  219. "at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  220. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  221. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  222. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  223. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  224. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  225. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  226. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
  227. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
  228. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  229. "Caused by: java.io.IOException: Connection refused: 10.0.0.87/10.0.0.87:9300",
  230. "Caused by: java.io.IOException: Connection refused",
  231. "at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
  232. "at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]",
  233. "at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]",
  234. "at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  235. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  236. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  237. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  238. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  239. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  240. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  241. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  242. "at java.lang.Thread.run(Thread.java:830) ~[?:?]"] }
  243. {"type": "server", "timestamp": "2020-02-19T18:22:07,020Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "failed to join {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=9, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, targetNode={es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}}]}",
  244. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [es02][10.0.7.10:9300][internal:cluster/coordination/join]",
  245. "Caused by: org.elasticsearch.transport.ConnectTransportException: [es01][10.0.0.87:9300] connect_exception",
  246. "at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:989) ~[elasticsearch-7.6.0.jar:7.6.0]",
  247. "at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:162) ~[elasticsearch-7.6.0.jar:7.6.0]",
  248. "at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  249. "at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
  250. "at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
  251. "at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
  252. "at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]",
  253. "at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  254. "at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[transport-netty4-client-7.6.0.jar:7.6.0]",
  255. "at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  256. "at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  257. "at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  258. "at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  259. "at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  260. "at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  261. "at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[netty-common-4.1.43.Final.jar:4.1.43.Final]",
  262. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  263. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  264. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  265. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  266. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  267. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  268. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
  269. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
  270. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  271. "Caused by: java.io.IOException: Connection refused: 10.0.0.87/10.0.0.87:9300",
  272. "Caused by: java.io.IOException: Connection refused",
  273. "at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
  274. "at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]",
  275. "at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]",
  276. "at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  277. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[netty-transport-4.1.43.Final.jar:4.1.43.Final]",
  278. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  279. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  280. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  281. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  282. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  283. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  284. "at java.lang.Thread.run(Thread.java:830) ~[?:?]"] }
  285. {"type": "server", "timestamp": "2020-02-19T18:22:13,738Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "last failed join attempt was 6.7s ago, failed to join {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976,
  286. ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=9, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, targetNode={es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}}]}",
  287. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [es02][10.0.7.10:9300][internal:cluster/coordination/join]",
  288. "Caused by: org.elasticsearch.transport.ConnectTransportException: [es01][10.0.0.87:9300] connect_exception",
  289. "at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:989) ~[elasticsearch-7.6.0.jar:7.6.0]",
  290. "at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:162) ~[elasticsearch-7.6.0.jar:7.6.0]",
  291. "at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  292. "at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]",
  293. "at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]",
  294. "at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]",
  295. "at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]",
  296. "at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.6.0.jar:7.6.0]",
  297. "at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]",
  298. "at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:577) ~[?:?]",
  299. "at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:570) ~[?:?]",
  300. "at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:549) ~[?:?]",
  301. "at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:490) ~[?:?]",
  302. "at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:615) ~[?:?]",
  303. "at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:608) ~[?:?]",
  304. "at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:117) ~[?:?]",
  305. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:321) ~[?:?]",
  306. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:337) ~[?:?]",
  307. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  308. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  309. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  310. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  311. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  312. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  313. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  314. "Caused by: java.io.IOException: Connection refused: 10.0.0.87/10.0.0.87:9300",
  315. "Caused by: java.io.IOException: Connection refused",
  316. "at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]",
  317. "at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]",
  318. "at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]",
  319. "at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:330) ~[?:?]",
  320. "at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:334) ~[?:?]",
  321. "at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:688) ~[?:?]",
  322. "at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) ~[?:?]",
  323. "at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) ~[?:?]",
  324. "at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) ~[?:?]",
  325. "at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) ~[?:?]",
  326. "at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[?:?]",
  327. "at java.lang.Thread.run(Thread.java:830) ~[?:?]"] }
  328. {"type": "server", "timestamp": "2020-02-19T18:22:13,740Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [SY7m05obQdCLexPLTWZFVQ, 55fLj3lWTTmWCqWUAk3vHg], have discovered [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, {es03}{uAqCgohVQGaEJNyR9kYwIg}{Mvlc3iK0QJ2uiV7HuJF_xg}{10.0.7.12}{10.0.7.12:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}, {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}] which is a quorum; discovery will continue using [10.0.7.9:9300, 10.0.7.11:9300] from hosts providers and [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state; node term 9, last-accepted version 0 in term 0" }
  329. {"type": "server", "timestamp": "2020-02-19T18:22:23,741Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "es-docker-cluster", "node.name": "es01", "message": "master not discovered or elected yet, an election requires 2 nodes with ids [SY7m05obQdCLexPLTWZFVQ, 55fLj3lWTTmWCqWUAk3vHg], have discovered [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}, {es03}{uAqCgohVQGaEJNyR9kYwIg}{Mvlc3iK0QJ2uiV7HuJF_xg}{10.0.7.12}{10.0.7.12:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}, {es02}{55fLj3lWTTmWCqWUAk3vHg}{FTOdPOtYRROvu_08j1NCJQ}{10.0.7.10}{10.0.7.10:9300}{dilm}{ml.machine_memory=4651646976, ml.max_open_jobs=20, xpack.installed=true}] which is a quorum; discovery will continue using [10.0.7.9:9300, 10.0.7.11:9300] from hosts providers and [{es01}{SY7m05obQdCLexPLTWZFVQ}{cHy-C6-_Smipq-ATuC8DUA}{10.0.0.87}{10.0.0.87:9300}{dilm}{ml.machine_memory=4651646976, xpack.installed=true, ml.max_open_jobs=20}] from last-known cluster state; node term 9, last-accepted version 0 in term 0" }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!