API tools faq
paste
Advertisement
Guest User

elasticsearch logs

a guest
Jul 1st, 2019
158
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
JSON 227.75 KB | None | 0 0
raw download clone embed print report
  1. OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
  2. {"type": "server", "timestamp": "2019-07-01T18:48:10,416+0000", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (fs-20c9ba59.efs.us-east-2.amazonaws.com://elasticsearch)]], net usable_space [8191.9pb], net total_space [8191.9pb], types [nfs4]"  }
  3. {"type": "server", "timestamp": "2019-07-01T18:48:10,419+0000", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "heap size [1.9gb], compressed ordinary object pointers [true]"  }
  4. {"type": "server", "timestamp": "2019-07-01T18:48:10,472+0000", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "node name [elasticsearch-master-1], node ID [oKYbqreMTH6SMgX9xNlR3g]"  }
  5. {"type": "server", "timestamp": "2019-07-01T18:48:10,472+0000", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "version[7.0.0], pid[1], build[default/docker/b7e28a7/2019-04-05T22:55:32.697037Z], OS[Linux/4.4.0-1079-aws/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/12/12+33]"  }
  6. {"type": "server", "timestamp": "2019-07-01T18:48:10,473+0000", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "JVM home [/usr/share/elasticsearch/jdk]"  }
  7. {"type": "server", "timestamp": "2019-07-01T18:48:10,473+0000", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "JVM arguments [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/tmp/elasticsearch-736422685958408947, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.locale.providers=COMPAT, -Dio.netty.allocator.type=unpooled, -Xmx2g, -Xms2g, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]"  }
  8. {"type": "server", "timestamp": "2019-07-01T18:48:16,624+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [aggs-matrix-stats]"  }
  9. {"type": "server", "timestamp": "2019-07-01T18:48:16,625+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [analysis-common]"  }
  10. {"type": "server", "timestamp": "2019-07-01T18:48:16,626+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [ingest-common]"  }
  11. {"type": "server", "timestamp": "2019-07-01T18:48:16,627+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [ingest-geoip]"  }
  12. {"type": "server", "timestamp": "2019-07-01T18:48:16,628+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [ingest-user-agent]"  }
  13. {"type": "server", "timestamp": "2019-07-01T18:48:16,631+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [lang-expression]"  }
  14. {"type": "server", "timestamp": "2019-07-01T18:48:16,632+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [lang-mustache]"  }
  15. {"type": "server", "timestamp": "2019-07-01T18:48:16,634+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [lang-painless]"  }
  16. {"type": "server", "timestamp": "2019-07-01T18:48:16,635+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [mapper-extras]"  }
  17. {"type": "server", "timestamp": "2019-07-01T18:48:16,636+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [parent-join]"  }
  18. {"type": "server", "timestamp": "2019-07-01T18:48:16,637+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [percolator]"  }
  19. {"type": "server", "timestamp": "2019-07-01T18:48:16,638+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [rank-eval]"  }
  20. {"type": "server", "timestamp": "2019-07-01T18:48:16,639+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [reindex]"  }
  21. {"type": "server", "timestamp": "2019-07-01T18:48:16,640+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [repository-url]"  }
  22. {"type": "server", "timestamp": "2019-07-01T18:48:16,641+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [transport-netty4]"  }
  23. {"type": "server", "timestamp": "2019-07-01T18:48:16,642+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-ccr]"  }
  24. {"type": "server", "timestamp": "2019-07-01T18:48:16,643+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-core]"  }
  25. {"type": "server", "timestamp": "2019-07-01T18:48:16,645+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-deprecation]"  }
  26. {"type": "server", "timestamp": "2019-07-01T18:48:16,647+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-graph]"  }
  27. {"type": "server", "timestamp": "2019-07-01T18:48:16,648+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-ilm]"  }
  28. {"type": "server", "timestamp": "2019-07-01T18:48:16,650+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-logstash]"  }
  29. {"type": "server", "timestamp": "2019-07-01T18:48:16,651+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-ml]"  }
  30. {"type": "server", "timestamp": "2019-07-01T18:48:16,651+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-monitoring]"  }
  31. {"type": "server", "timestamp": "2019-07-01T18:48:16,653+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-rollup]"  }
  32. {"type": "server", "timestamp": "2019-07-01T18:48:16,654+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-security]"  }
  33. {"type": "server", "timestamp": "2019-07-01T18:48:16,654+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-sql]"  }
  34. {"type": "server", "timestamp": "2019-07-01T18:48:16,656+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "loaded module [x-pack-watcher]"  }
  35. {"type": "server", "timestamp": "2019-07-01T18:48:16,658+0000", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "no plugins loaded"  }
  36. {"type": "server", "timestamp": "2019-07-01T18:48:29,172+0000", "level": "INFO", "component": "o.e.x.s.a.s.FileRolesStore", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]"  }
  37. {"type": "server", "timestamp": "2019-07-01T18:48:31,917+0000", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "[controller/86] [Main.cc@109] controller (64 bit): Version 7.0.0 (Build cdaa022645f38d) Copyright (c) 2019 Elasticsearch BV"  }
  38. {"type": "server", "timestamp": "2019-07-01T18:48:33,622+0000", "level": "DEBUG", "component": "o.e.a.ActionModule", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "Using REST wrapper from plugin org.elasticsearch.xpack.security.Security"  }
  39. {"type": "server", "timestamp": "2019-07-01T18:48:34,456+0000", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "using discovery type [zen] and seed hosts providers [settings]"  }
  40. {"type": "server", "timestamp": "2019-07-01T18:48:37,000+0000", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "initialized"  }
  41. {"type": "server", "timestamp": "2019-07-01T18:48:37,004+0000", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "starting ..."  }
  42. {"type": "server", "timestamp": "2019-07-01T18:48:37,350+0000", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "publish_address {172.31.29.199:9300}, bound_addresses {0.0.0.0:9300}"  }
  43. {"type": "server", "timestamp": "2019-07-01T18:48:37,371+0000", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks"  }
  44. {"type": "server", "timestamp": "2019-07-01T18:48:39,374+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  45. "stacktrace": ["org.elasticsearch.transport.NodeNotConnectedException: [elasticsearch-master-2][172.31.11.195:9300] Node not connected",
  46. "at org.elasticsearch.transport.ConnectionManager.getConnection(ConnectionManager.java:151) ~[elasticsearch-7.0.0.jar:7.0.0]",
  47. "at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:557) ~[elasticsearch-7.0.0.jar:7.0.0]",
  48. "at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:529) [elasticsearch-7.0.0.jar:7.0.0]",
  49. "at org.elasticsearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:270) [elasticsearch-7.0.0.jar:7.0.0]",
  50. "at org.elasticsearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:203) [elasticsearch-7.0.0.jar:7.0.0]",
  51. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:135) [elasticsearch-7.0.0.jar:7.0.0]",
  52. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  53. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  54. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  55. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  56. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) [elasticsearch-7.0.0.jar:7.0.0]",
  57. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  58. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  59. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  60. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  61. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  62. {"type": "server", "timestamp": "2019-07-01T18:48:39,448+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  63. "stacktrace": ["org.elasticsearch.transport.NodeNotConnectedException: [elasticsearch-master-2][172.31.11.195:9300] Node not connected",
  64. "at org.elasticsearch.transport.ConnectionManager.getConnection(ConnectionManager.java:151) ~[elasticsearch-7.0.0.jar:7.0.0]",
  65. "at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:557) ~[elasticsearch-7.0.0.jar:7.0.0]",
  66. "at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:529) [elasticsearch-7.0.0.jar:7.0.0]",
  67. "at org.elasticsearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:270) [elasticsearch-7.0.0.jar:7.0.0]",
  68. "at org.elasticsearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:203) [elasticsearch-7.0.0.jar:7.0.0]",
  69. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:135) [elasticsearch-7.0.0.jar:7.0.0]",
  70. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  71. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  72. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  73. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  74. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) [elasticsearch-7.0.0.jar:7.0.0]",
  75. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  76. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  77. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  78. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  79. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  80. {"type": "server", "timestamp": "2019-07-01T18:48:39,493+0000", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "setting initial configuration to VotingConfiguration{oKYbqreMTH6SMgX9xNlR3g,{bootstrap-placeholder}-elasticsearch-master-0,xiuloUmzR3ipJJ-rw6P4Wg}"  }
  81. {"type": "server", "timestamp": "2019-07-01T18:48:40,046+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=2, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  82. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  83. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}",
  84. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  85. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  86. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  87. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  88. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  89. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  90. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  91. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  92. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) [elasticsearch-7.0.0.jar:7.0.0]",
  93. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  94. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  95. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  96. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  97. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  98. {"type": "server", "timestamp": "2019-07-01T18:48:40,264+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=3, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  99. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.17.0.2:9300][internal:cluster/coordination/join]",
  100. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  101. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) ~[elasticsearch-7.0.0.jar:7.0.0]",
  102. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) ~[elasticsearch-7.0.0.jar:7.0.0]",
  103. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  104. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  105. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  106. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  107. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  108. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  109. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) ~[elasticsearch-7.0.0.jar:7.0.0]",
  110. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  111. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  112. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  113. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  114. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  115. {"type": "server", "timestamp": "2019-07-01T18:48:40,329+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} elect leader, {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 4, version: 1, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, added {{elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true},}"  }
  116. {"type": "server", "timestamp": "2019-07-01T18:48:40,885+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, added {{elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true},}, term: 4, version: 1, reason: Publication{term=4, version=1}"  }
  117. {"type": "server", "timestamp": "2019-07-01T18:48:41,047+0000", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "publish_address {172.31.29.199:9200}, bound_addresses {0.0.0.0:9200}"  }
  118. {"type": "server", "timestamp": "2019-07-01T18:48:41,048+0000", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "started"  }
  119. {"type": "server", "timestamp": "2019-07-01T18:48:41,224+0000", "level": "WARN", "component": "o.e.x.s.a.s.m.NativeRoleMappingStore", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "Failed to clear cache for realms [[]]"  }
  120. {"type": "server", "timestamp": "2019-07-01T18:48:41,461+0000", "level": "INFO", "component": "o.e.g.GatewayService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "recovered [0] indices into cluster_state"  }
  121. {"type": "server", "timestamp": "2019-07-01T18:48:42,648+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.watches] for index patterns [.watches*]"  }
  122. {"type": "server", "timestamp": "2019-07-01T18:48:42,843+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.triggered_watches] for index patterns [.triggered_watches*]"  }
  123. {"type": "server", "timestamp": "2019-07-01T18:48:43,185+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.watch-history-9] for index patterns [.watcher-history-9*]"  }
  124. {"type": "server", "timestamp": "2019-07-01T18:48:43,403+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]"  }
  125. {"type": "server", "timestamp": "2019-07-01T18:48:43,584+0000", "level": "WARN", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failing [create-index-template [.monitoring-logstash], cause [api]]: failed to commit cluster state version [7]" ,
  126. "stacktrace": ["org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  127. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  128. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  129. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  130. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  131. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  132. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  133. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  134. "at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:88) ~[elasticsearch-7.0.0.jar:7.0.0]",
  135. "at org.elasticsearch.cluster.coordination.Coordinator.cancelActivePublication(Coordinator.java:1086) ~[elasticsearch-7.0.0.jar:7.0.0]",
  136. "at org.elasticsearch.cluster.coordination.Coordinator.becomeCandidate(Coordinator.java:529) ~[elasticsearch-7.0.0.jar:7.0.0]",
  137. "at org.elasticsearch.cluster.coordination.Coordinator.joinLeaderInTerm(Coordinator.java:442) ~[elasticsearch-7.0.0.jar:7.0.0]",
  138. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:135) ~[elasticsearch-7.0.0.jar:7.0.0]",
  139. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  140. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  141. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  142. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  143. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  144. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  145. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  146. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  147. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  148. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  149. "Caused by: org.elasticsearch.ElasticsearchException: publication cancelled before committing: become candidate: joinLeaderInTerm",
  150. "at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:85) ~[elasticsearch-7.0.0.jar:7.0.0]",
  151. "... 14 more"] }
  152. {"type": "server", "timestamp": "2019-07-01T18:48:43,587+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to put template [.monitoring-logstash]" ,
  153. "stacktrace": ["org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  154. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  155. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  156. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  157. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  158. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  159. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  160. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  161. "at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:88) ~[elasticsearch-7.0.0.jar:7.0.0]",
  162. "at org.elasticsearch.cluster.coordination.Coordinator.cancelActivePublication(Coordinator.java:1086) ~[elasticsearch-7.0.0.jar:7.0.0]",
  163. "at org.elasticsearch.cluster.coordination.Coordinator.becomeCandidate(Coordinator.java:529) ~[elasticsearch-7.0.0.jar:7.0.0]",
  164. "at org.elasticsearch.cluster.coordination.Coordinator.joinLeaderInTerm(Coordinator.java:442) ~[elasticsearch-7.0.0.jar:7.0.0]",
  165. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:135) ~[elasticsearch-7.0.0.jar:7.0.0]",
  166. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  167. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  168. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  169. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  170. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  171. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  172. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  173. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  174. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  175. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  176. "Caused by: org.elasticsearch.ElasticsearchException: publication cancelled before committing: become candidate: joinLeaderInTerm",
  177. "at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:85) ~[elasticsearch-7.0.0.jar:7.0.0]",
  178. "... 14 more"] }
  179. {"type": "server", "timestamp": "2019-07-01T18:48:43,607+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 4, version: 6, reason: becoming candidate: joinLeaderInTerm"  }
  180. {"type": "server", "timestamp": "2019-07-01T18:48:43,611+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=5, lastAcceptedTerm=4, lastAcceptedVersion=7, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  181. "stacktrace": ["org.elasticsearch.transport.NodeNotConnectedException: [elasticsearch-master-0][172.31.33.10:9300] Node not connected",
  182. "at org.elasticsearch.transport.ConnectionManager.getConnection(ConnectionManager.java:151) ~[elasticsearch-7.0.0.jar:7.0.0]",
  183. "at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:557) ~[elasticsearch-7.0.0.jar:7.0.0]",
  184. "at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:529) [elasticsearch-7.0.0.jar:7.0.0]",
  185. "at org.elasticsearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:270) [elasticsearch-7.0.0.jar:7.0.0]",
  186. "at org.elasticsearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:203) [elasticsearch-7.0.0.jar:7.0.0]",
  187. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:135) [elasticsearch-7.0.0.jar:7.0.0]",
  188. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  189. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  190. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  191. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  192. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) [elasticsearch-7.0.0.jar:7.0.0]",
  193. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  194. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  195. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  196. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  197. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  198. {"type": "server", "timestamp": "2019-07-01T18:48:43,604+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master could not publish cluster state or stepped down before publishing action [indices:admin/template/put], scheduling a retry" ,
  199. "stacktrace": ["org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  200. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  201. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  202. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  203. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  204. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  205. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  206. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  207. "at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:88) ~[elasticsearch-7.0.0.jar:7.0.0]",
  208. "at org.elasticsearch.cluster.coordination.Coordinator.cancelActivePublication(Coordinator.java:1086) ~[elasticsearch-7.0.0.jar:7.0.0]",
  209. "at org.elasticsearch.cluster.coordination.Coordinator.becomeCandidate(Coordinator.java:529) ~[elasticsearch-7.0.0.jar:7.0.0]",
  210. "at org.elasticsearch.cluster.coordination.Coordinator.joinLeaderInTerm(Coordinator.java:442) ~[elasticsearch-7.0.0.jar:7.0.0]",
  211. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:135) ~[elasticsearch-7.0.0.jar:7.0.0]",
  212. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  213. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  214. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  215. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  216. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  217. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  218. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  219. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  220. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  221. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  222. "Caused by: org.elasticsearch.ElasticsearchException: publication cancelled before committing: become candidate: joinLeaderInTerm",
  223. "at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:85) ~[elasticsearch-7.0.0.jar:7.0.0]",
  224. "... 14 more"] }
  225. {"type": "server", "timestamp": "2019-07-01T18:48:43,642+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to put template [.monitoring-es]" ,
  226. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [create-index-template [.monitoring-es], cause [api]]"] }
  227. {"type": "server", "timestamp": "2019-07-01T18:48:43,643+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master could not publish cluster state or stepped down before publishing action [indices:admin/template/put], scheduling a retry" ,
  228. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [create-index-template [.monitoring-es], cause [api]]"] }
  229. {"type": "server", "timestamp": "2019-07-01T18:48:43,644+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=5, lastAcceptedTerm=4, lastAcceptedVersion=7, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  230. "stacktrace": ["org.elasticsearch.transport.NodeNotConnectedException: [elasticsearch-master-0][172.31.33.10:9300] Node not connected",
  231. "at org.elasticsearch.transport.ConnectionManager.getConnection(ConnectionManager.java:151) ~[elasticsearch-7.0.0.jar:7.0.0]",
  232. "at org.elasticsearch.transport.TransportService.getConnection(TransportService.java:557) ~[elasticsearch-7.0.0.jar:7.0.0]",
  233. "at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:529) [elasticsearch-7.0.0.jar:7.0.0]",
  234. "at org.elasticsearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:270) [elasticsearch-7.0.0.jar:7.0.0]",
  235. "at org.elasticsearch.cluster.coordination.JoinHelper.sendJoinRequest(JoinHelper.java:203) [elasticsearch-7.0.0.jar:7.0.0]",
  236. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:135) [elasticsearch-7.0.0.jar:7.0.0]",
  237. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  238. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  239. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  240. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  241. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) [elasticsearch-7.0.0.jar:7.0.0]",
  242. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  243. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  244. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  245. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  246. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  247. {"type": "server", "timestamp": "2019-07-01T18:48:43,648+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to put template [.monitoring-beats]" ,
  248. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [create-index-template [.monitoring-beats], cause [api]]"] }
  249. {"type": "server", "timestamp": "2019-07-01T18:48:43,649+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master could not publish cluster state or stepped down before publishing action [indices:admin/template/put], scheduling a retry" ,
  250. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [create-index-template [.monitoring-beats], cause [api]]"] }
  251. {"type": "server", "timestamp": "2019-07-01T18:48:43,652+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to put template [.monitoring-alerts-7]" ,
  252. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [create-index-template [.monitoring-alerts-7], cause [api]]"] }
  253. {"type": "server", "timestamp": "2019-07-01T18:48:43,655+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master could not publish cluster state or stepped down before publishing action [indices:admin/template/put], scheduling a retry" ,
  254. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [create-index-template [.monitoring-alerts-7], cause [api]]"] }
  255. {"type": "server", "timestamp": "2019-07-01T18:48:43,660+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to put template [.monitoring-kibana]" ,
  256. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [create-index-template [.monitoring-kibana], cause [api]]"] }
  257. {"type": "server", "timestamp": "2019-07-01T18:48:43,661+0000", "level": "DEBUG", "component": "o.e.a.a.i.t.p.TransportPutIndexTemplateAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master could not publish cluster state or stepped down before publishing action [indices:admin/template/put], scheduling a retry" ,
  258. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [create-index-template [.monitoring-kibana], cause [api]]"] }
  259. {"type": "server", "timestamp": "2019-07-01T18:48:43,662+0000", "level": "ERROR", "component": "o.e.x.s.a.TokenService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unable to install token metadata" ,
  260. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [install-token-metadata]"] }
  261. {"type": "server", "timestamp": "2019-07-01T18:48:43,666+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  262. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  263. {"type": "server", "timestamp": "2019-07-01T18:48:43,677+0000", "level": "DEBUG", "component": "o.e.a.i.PutPipelineTransportAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master could not publish cluster state or stepped down before publishing action [cluster:admin/ingest/pipeline/put], scheduling a retry" ,
  264. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [put-pipeline-xpack_monitoring_7]"] }
  265. {"type": "server", "timestamp": "2019-07-01T18:48:43,691+0000", "level": "DEBUG", "component": "o.e.a.i.PutPipelineTransportAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master could not publish cluster state or stepped down before publishing action [cluster:admin/ingest/pipeline/put], scheduling a retry" ,
  266. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [put-pipeline-xpack_monitoring_6]"] }
  267. {"type": "server", "timestamp": "2019-07-01T18:48:43,700+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  268. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  269. {"type": "server", "timestamp": "2019-07-01T18:48:43,703+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  270. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  271. {"type": "server", "timestamp": "2019-07-01T18:48:43,706+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  272. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  273. {"type": "server", "timestamp": "2019-07-01T18:48:43,709+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  274. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  275. {"type": "server", "timestamp": "2019-07-01T18:48:44,332+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} elect leader, {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 6, version: 8, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}"  }
  276. {"type": "server", "timestamp": "2019-07-01T18:48:44,525+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, term: 6, version: 8, reason: Publication{term=6, version=8}"  }
  277. {"type": "server", "timestamp": "2019-07-01T18:48:44,584+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]"  }
  278. {"type": "server", "timestamp": "2019-07-01T18:48:44,761+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]"  }
  279. {"type": "server", "timestamp": "2019-07-01T18:48:45,019+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.monitoring-es] for index patterns [.monitoring-es-7-*]"  }
  280. {"type": "server", "timestamp": "2019-07-01T18:48:45,248+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]"  }
  281. {"type": "server", "timestamp": "2019-07-01T18:48:45,393+0000", "level": "INFO", "component": "o.e.c.m.MetaDataIndexTemplateService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding template [.monitoring-beats] for index patterns [.monitoring-beats-7-*]"  }
  282. {"type": "server", "timestamp": "2019-07-01T18:48:45,937+0000", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "license [b21735fa-cf7d-4e71-9fc8-3c20a1fcf86e] mode [basic] - valid"  }
  283. {"type": "server", "timestamp": "2019-07-01T18:48:46,193+0000", "level": "ERROR", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to set monitoring pipeline [xpack_monitoring_7]"  }
  284. {"type": "server", "timestamp": "2019-07-01T18:48:46,342+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 6, version: 15, reason: becoming candidate: Publication.onCompletion(false)"  }
  285. {"type": "server", "timestamp": "2019-07-01T18:48:46,346+0000", "level": "WARN", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failing [put-pipeline-xpack_monitoring_6]: failed to commit cluster state version [16]" ,
  286. "stacktrace": ["org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  287. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  288. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  289. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  290. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  291. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  292. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  293. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  294. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  295. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  296. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  297. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  298. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  299. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  300. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  301. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  302. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  303. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  304. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  305. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  306. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  307. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  308. "... 11 more"] }
  309. {"type": "server", "timestamp": "2019-07-01T18:48:46,347+0000", "level": "DEBUG", "component": "o.e.a.i.PutPipelineTransportAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master could not publish cluster state or stepped down before publishing action [cluster:admin/ingest/pipeline/put], scheduling a retry" ,
  310. "stacktrace": ["org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  311. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  312. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  313. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  314. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  315. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  316. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  317. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  318. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  319. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  320. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  321. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  322. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  323. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  324. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  325. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  326. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  327. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  328. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  329. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  330. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  331. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  332. "... 11 more"] }
  333. {"type": "server", "timestamp": "2019-07-01T18:48:46,390+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  334. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  335. {"type": "server", "timestamp": "2019-07-01T18:48:46,393+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  336. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  337. {"type": "server", "timestamp": "2019-07-01T18:48:46,395+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  338. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  339. {"type": "server", "timestamp": "2019-07-01T18:48:46,397+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  340. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  341. {"type": "server", "timestamp": "2019-07-01T18:48:46,398+0000", "level": "ERROR", "component": "o.e.l.StartupSelfGeneratedLicenseTask", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "unexpected failure during [maybe generate license for cluster]" ,
  342. "stacktrace": ["org.elasticsearch.cluster.NotMasterException: no longer master. source: [maybe generate license for cluster]"] }
  343. {"type": "server", "timestamp": "2019-07-01T18:48:48,708+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=10, lastAcceptedTerm=6, lastAcceptedVersion=16, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  344. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  345. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: incoming last accepted version 16 of join higher than current last accepted version 15 in term 6",
  346. "at org.elasticsearch.cluster.coordination.CoordinationState.handleJoin(CoordinationState.java:239) ~[elasticsearch-7.0.0.jar:7.0.0]",
  347. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoin(Coordinator.java:940) ~[elasticsearch-7.0.0.jar:7.0.0]",
  348. "at java.util.Optional.ifPresent(Optional.java:183) ~[?:?]",
  349. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:512) ~[elasticsearch-7.0.0.jar:7.0.0]",
  350. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  351. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  352. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  353. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  354. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  355. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  356. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  357. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  358. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  359. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  360. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  361. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  362. {"type": "server", "timestamp": "2019-07-01T18:48:49,249+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=8, lastAcceptedTerm=6, lastAcceptedVersion=16, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  363. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.17.0.2:9300][internal:cluster/coordination/join]",
  364. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}",
  365. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) ~[elasticsearch-7.0.0.jar:7.0.0]",
  366. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) ~[elasticsearch-7.0.0.jar:7.0.0]",
  367. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  368. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  369. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  370. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  371. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  372. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  373. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  374. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  375. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  376. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  377. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  378. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  379. {"type": "server", "timestamp": "2019-07-01T18:48:49,392+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=12, lastAcceptedTerm=6, lastAcceptedVersion=16, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  380. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  381. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: incoming last accepted version 16 of join higher than current last accepted version 15 in term 6",
  382. "at org.elasticsearch.cluster.coordination.CoordinationState.handleJoin(CoordinationState.java:239) ~[elasticsearch-7.0.0.jar:7.0.0]",
  383. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoin(Coordinator.java:940) ~[elasticsearch-7.0.0.jar:7.0.0]",
  384. "at java.util.Optional.ifPresent(Optional.java:183) ~[?:?]",
  385. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:512) ~[elasticsearch-7.0.0.jar:7.0.0]",
  386. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  387. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  388. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  389. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  390. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  391. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  392. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  393. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  394. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  395. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  396. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  397. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  398. {"type": "server", "timestamp": "2019-07-01T18:48:50,225+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=13, lastAcceptedTerm=6, lastAcceptedVersion=16, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  399. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-0][172.17.0.2:9300][internal:cluster/coordination/join]",
  400. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: incoming last accepted term 6 of join higher than current last accepted term 0",
  401. "at org.elasticsearch.cluster.coordination.CoordinationState.handleJoin(CoordinationState.java:230) ~[elasticsearch-7.0.0.jar:7.0.0]",
  402. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoin(Coordinator.java:940) ~[elasticsearch-7.0.0.jar:7.0.0]",
  403. "at java.util.Optional.ifPresent(Optional.java:183) ~[?:?]",
  404. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:512) ~[elasticsearch-7.0.0.jar:7.0.0]",
  405. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  406. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  407. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  408. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  409. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  410. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  411. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  412. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  413. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  414. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  415. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  416. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  417. {"type": "server", "timestamp": "2019-07-01T18:48:51,342+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=11, lastAcceptedTerm=6, lastAcceptedVersion=16, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  418. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.17.0.2:9300][internal:cluster/coordination/join]",
  419. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}",
  420. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) ~[elasticsearch-7.0.0.jar:7.0.0]",
  421. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) ~[elasticsearch-7.0.0.jar:7.0.0]",
  422. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  423. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  424. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  425. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  426. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  427. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  428. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  429. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  430. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  431. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  432. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  433. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  434. {"type": "server", "timestamp": "2019-07-01T18:48:51,678+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 15, version: 17, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}"  }
  435. {"type": "server", "timestamp": "2019-07-01T18:48:52,294+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, term: 15, version: 17, reason: Publication{term=15, version=17}"  }
  436. {"type": "server", "timestamp": "2019-07-01T18:48:52,325+0000", "level": "INFO", "component": "o.e.x.i.a.TransportPutLifecycleAction", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "adding index lifecycle policy [watch-history-ilm-policy]"  }
  437. {"type": "server", "timestamp": "2019-07-01T18:48:54,435+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 15, version: 19, reason: becoming candidate: joinLeaderInTerm"  }
  438. {"type": "server", "timestamp": "2019-07-01T18:48:54,592+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=16, lastAcceptedTerm=15, lastAcceptedVersion=19, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  439. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.17.0.2:9300][internal:cluster/coordination/join]",
  440. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  441. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) ~[elasticsearch-7.0.0.jar:7.0.0]",
  442. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) ~[elasticsearch-7.0.0.jar:7.0.0]",
  443. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  444. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  445. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  446. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  447. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  448. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  449. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) ~[elasticsearch-7.0.0.jar:7.0.0]",
  450. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  451. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  452. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  453. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  454. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  455. {"type": "server", "timestamp": "2019-07-01T18:48:54,607+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 17, version: 20, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}"  }
  456. {"type": "server", "timestamp": "2019-07-01T18:48:54,811+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, term: 17, version: 20, reason: Publication{term=17, version=20}"  }
  457. {"type": "server", "timestamp": "2019-07-01T18:48:58,716+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 17, version: 20, reason: becoming candidate: joinLeaderInTerm"  }
  458. {"type": "server", "timestamp": "2019-07-01T18:48:58,815+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=19, lastAcceptedTerm=17, lastAcceptedVersion=20, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  459. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.17.0.2:9300][internal:cluster/coordination/join]",
  460. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  461. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) ~[elasticsearch-7.0.0.jar:7.0.0]",
  462. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) ~[elasticsearch-7.0.0.jar:7.0.0]",
  463. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  464. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  465. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  466. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  467. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  468. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  469. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) ~[elasticsearch-7.0.0.jar:7.0.0]",
  470. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  471. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  472. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  473. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  474. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  475. {"type": "server", "timestamp": "2019-07-01T18:48:58,829+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 20, version: 21, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}"  }
  476. {"type": "server", "timestamp": "2019-07-01T18:48:59,045+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, term: 20, version: 21, reason: Publication{term=20, version=21}"  }
  477. {"type": "server", "timestamp": "2019-07-01T18:49:00,827+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 20, version: 21, reason: becoming candidate: joinLeaderInTerm"  }
  478. {"type": "server", "timestamp": "2019-07-01T18:49:00,978+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} elect leader, {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 22, version: 22, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}"  }
  479. {"type": "server", "timestamp": "2019-07-01T18:49:01,246+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=21, lastAcceptedTerm=20, lastAcceptedVersion=21, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  480. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  481. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: node is no longer master for term 22 while handling publication",
  482. "at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1010) ~[elasticsearch-7.0.0.jar:7.0.0]",
  483. "at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:252) ~[elasticsearch-7.0.0.jar:7.0.0]",
  484. "at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:238) ~[elasticsearch-7.0.0.jar:7.0.0]",
  485. "at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:142) ~[elasticsearch-7.0.0.jar:7.0.0]",
  486. "at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-7.0.0.jar:7.0.0]",
  487. "at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) ~[elasticsearch-7.0.0.jar:7.0.0]",
  488. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  489. "at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) ~[elasticsearch-7.0.0.jar:7.0.0]",
  490. "at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) ~[elasticsearch-7.0.0.jar:7.0.0]",
  491. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  492. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  493. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  494. {"type": "server", "timestamp": "2019-07-01T18:49:01,261+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, term: 22, version: 22, reason: Publication{term=22, version=22}"  }
  495. {"type": "server", "timestamp": "2019-07-01T18:49:04,680+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 22, version: 23, reason: becoming candidate: joinLeaderInTerm"  }
  496. {"type": "server", "timestamp": "2019-07-01T18:49:04,692+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=23, lastAcceptedTerm=22, lastAcceptedVersion=23, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  497. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-0][172.17.0.2:9300][internal:cluster/coordination/join]",
  498. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: incoming last accepted term 22 of join higher than current last accepted term 0",
  499. "at org.elasticsearch.cluster.coordination.CoordinationState.handleJoin(CoordinationState.java:230) ~[elasticsearch-7.0.0.jar:7.0.0]",
  500. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoin(Coordinator.java:940) ~[elasticsearch-7.0.0.jar:7.0.0]",
  501. "at java.util.Optional.ifPresent(Optional.java:183) ~[?:?]",
  502. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:512) ~[elasticsearch-7.0.0.jar:7.0.0]",
  503. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  504. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  505. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  506. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  507. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  508. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  509. "at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  510. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  511. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  512. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  513. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  514. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  515. {"type": "server", "timestamp": "2019-07-01T18:49:05,405+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=24, lastAcceptedTerm=22, lastAcceptedVersion=23, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  516. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  517. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  518. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  519. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  520. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  521. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  522. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  523. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  524. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  525. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  526. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  527. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  528. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  529. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  530. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  531. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  532. {"type": "server", "timestamp": "2019-07-01T18:49:06,145+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=25, lastAcceptedTerm=22, lastAcceptedVersion=23, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  533. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  534. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  535. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  536. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  537. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  538. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  539. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  540. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  541. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  542. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  543. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  544. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  545. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  546. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  547. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  548. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  549. {"type": "server", "timestamp": "2019-07-01T18:49:06,768+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=26, lastAcceptedTerm=22, lastAcceptedVersion=23, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  550. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  551. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  552. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  553. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  554. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  555. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  556. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  557. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  558. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  559. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  560. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  561. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  562. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  563. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  564. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  565. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  566. {"type": "server", "timestamp": "2019-07-01T18:49:06,783+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} elect leader, {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 28, version: 24, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}"  }
  567. {"type": "server", "timestamp": "2019-07-01T18:49:07,071+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, term: 28, version: 24, reason: Publication{term=28, version=24}"  }
  568. {"type": "server", "timestamp": "2019-07-01T18:49:10,213+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 28, version: 24, reason: becoming candidate: joinLeaderInTerm"  }
  569. {"type": "server", "timestamp": "2019-07-01T18:49:10,345+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=29, lastAcceptedTerm=28, lastAcceptedVersion=24, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  570. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.17.0.2:9300][internal:cluster/coordination/join]",
  571. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  572. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) ~[elasticsearch-7.0.0.jar:7.0.0]",
  573. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) ~[elasticsearch-7.0.0.jar:7.0.0]",
  574. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  575. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  576. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  577. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  578. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  579. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  580. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) ~[elasticsearch-7.0.0.jar:7.0.0]",
  581. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  582. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  583. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  584. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  585. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  586. {"type": "server", "timestamp": "2019-07-01T18:49:10,871+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=30, lastAcceptedTerm=28, lastAcceptedVersion=24, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  587. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  588. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  589. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  590. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  591. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  592. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  593. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  594. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  595. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  596. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  597. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  598. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  599. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  600. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  601. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  602. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  603. {"type": "server", "timestamp": "2019-07-01T18:49:11,585+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=31, lastAcceptedTerm=28, lastAcceptedVersion=24, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  604. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  605. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  606. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  607. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  608. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  609. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  610. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  611. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  612. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  613. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  614. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  615. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  616. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  617. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  618. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  619. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  620. {"type": "server", "timestamp": "2019-07-01T18:49:11,596+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 32, version: 25, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}"  }
  621. {"type": "server", "timestamp": "2019-07-01T18:49:11,829+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, term: 32, version: 25, reason: Publication{term=32, version=25}"  }
  622. {"type": "server", "timestamp": "2019-07-01T18:49:12,368+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "node-join[{elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} join existing leader], term: 32, version: 26, reason: added {{elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true},}"  }
  623. {"type": "server", "timestamp": "2019-07-01T18:49:12,447+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "added {{elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true},}, term: 32, version: 26, reason: Publication{term=32, version=26}"  }
  624. {"type": "server", "timestamp": "2019-07-01T18:49:13,349+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 32, version: 27, reason: becoming candidate: joinLeaderInTerm"  }
  625. {"type": "server", "timestamp": "2019-07-01T18:49:13,461+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=34, lastAcceptedTerm=32, lastAcceptedVersion=27, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  626. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  627. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  628. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  629. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  630. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  631. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  632. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  633. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  634. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  635. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  636. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  637. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  638. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  639. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  640. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  641. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  642. {"type": "server", "timestamp": "2019-07-01T18:49:13,486+0000", "level": "INFO", "component": "o.e.c.s.MasterService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "elected-as-master ([2] nodes joined)[{elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} elect leader, {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} elect leader, _BECOME_MASTER_TASK_, _FINISH_ELECTION_], term: 35, version: 28, reason: master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}"  }
  643. {"type": "server", "timestamp": "2019-07-01T18:49:13,729+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [], current [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}]}, term: 35, version: 28, reason: Publication{term=35, version=28}"  }
  644. {"type": "server", "timestamp": "2019-07-01T18:49:14,374+0000", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "master node changed {previous [{elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}], current []}, term: 35, version: 28, reason: becoming candidate: joinLeaderInTerm"  }
  645. {"type": "server", "timestamp": "2019-07-01T18:49:14,757+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=36, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  646. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  647. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  648. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  649. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  650. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  651. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  652. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  653. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  654. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  655. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  656. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  657. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  658. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  659. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  660. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  661. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  662. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  663. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  664. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  665. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  666. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  667. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  668. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  669. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  670. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  671. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  672. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  673. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  674. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  675. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  676. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  677. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  678. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  679. "at java.lang.Thread.run(Thread.java:835) ~[?:?]"] }
  680. {"type": "server", "timestamp": "2019-07-01T18:49:15,184+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=38, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  681. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.17.0.2:9300][internal:cluster/coordination/join]",
  682. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  683. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) ~[elasticsearch-7.0.0.jar:7.0.0]",
  684. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) ~[elasticsearch-7.0.0.jar:7.0.0]",
  685. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  686. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  687. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  688. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  689. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  690. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  691. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) ~[elasticsearch-7.0.0.jar:7.0.0]",
  692. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  693. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  694. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  695. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  696. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  697. {"type": "server", "timestamp": "2019-07-01T18:49:15,707+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=39, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  698. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  699. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  700. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  701. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  702. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  703. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  704. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  705. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  706. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  707. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  708. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  709. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  710. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  711. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  712. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  713. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  714. {"type": "server", "timestamp": "2019-07-01T18:49:16,390+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=41, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  715. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  716. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  717. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  718. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  719. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  720. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  721. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  722. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  723. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  724. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  725. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  726. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  727. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  728. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  729. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  730. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  731. {"type": "server", "timestamp": "2019-07-01T18:49:17,008+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=43, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  732. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  733. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  734. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  735. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  736. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  737. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  738. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  739. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  740. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  741. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  742. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  743. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  744. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  745. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  746. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  747. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  748. {"type": "server", "timestamp": "2019-07-01T18:49:17,717+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=45, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  749. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  750. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  751. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  752. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  753. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  754. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  755. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  756. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  757. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  758. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  759. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  760. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  761. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  762. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  763. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  764. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  765. {"type": "server", "timestamp": "2019-07-01T18:49:18,372+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=47, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  766. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  767. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  768. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  769. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  770. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  771. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  772. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  773. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  774. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  775. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  776. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  777. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  778. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  779. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  780. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  781. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  782. {"type": "server", "timestamp": "2019-07-01T18:49:19,559+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=48, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  783. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  784. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  785. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  786. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  787. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  788. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  789. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  790. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  791. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  792. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  793. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  794. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  795. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  796. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  797. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  798. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  799. {"type": "server", "timestamp": "2019-07-01T18:49:20,267+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=51, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  800. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  801. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  802. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  803. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  804. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  805. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  806. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  807. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  808. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  809. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  810. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  811. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  812. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  813. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  814. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  815. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  816. {"type": "server", "timestamp": "2019-07-01T18:49:20,805+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=53, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  817. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  818. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  819. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  820. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  821. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  822. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  823. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  824. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  825. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  826. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  827. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  828. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  829. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  830. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  831. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  832. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  833. {"type": "server", "timestamp": "2019-07-01T18:49:21,531+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=54, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  834. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  835. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  836. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  837. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  838. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  839. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  840. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  841. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  842. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  843. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  844. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  845. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  846. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  847. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  848. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  849. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  850. {"type": "server", "timestamp": "2019-07-01T18:49:21,687+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=56, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  851. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  852. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  853. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  854. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  855. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  856. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  857. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  858. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  859. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  860. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  861. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  862. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  863. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  864. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  865. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  866. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  867. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  868. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  869. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  870. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  871. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  872. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  873. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  874. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  875. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  876. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  877. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  878. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  879. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  880. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  881. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  882. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  883. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  884. "at java.lang.Thread.run(Thread.java:835) ~[?:?]"] }
  885. {"type": "server", "timestamp": "2019-07-01T18:49:21,805+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=57, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  886. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  887. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: became follower",
  888. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.lambda$close$3(JoinHelper.java:468) [elasticsearch-7.0.0.jar:7.0.0]",
  889. "at java.util.HashMap$Values.forEach(HashMap.java:976) [?:?]",
  890. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.close(JoinHelper.java:468) [elasticsearch-7.0.0.jar:7.0.0]",
  891. "at org.elasticsearch.cluster.coordination.Coordinator.becomeFollower(Coordinator.java:601) [elasticsearch-7.0.0.jar:7.0.0]",
  892. "at org.elasticsearch.cluster.coordination.Coordinator.onFollowerCheckRequest(Coordinator.java:237) [elasticsearch-7.0.0.jar:7.0.0]",
  893. "at org.elasticsearch.cluster.coordination.FollowersChecker$2.doRun(FollowersChecker.java:187) [elasticsearch-7.0.0.jar:7.0.0]",
  894. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  895. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  896. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  897. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  898. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  899. {"type": "server", "timestamp": "2019-07-01T18:49:22,021+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=58, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  900. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  901. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  902. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  903. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  904. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  905. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  906. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  907. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  908. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  909. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  910. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  911. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  912. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  913. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  914. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  915. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  916. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  917. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  918. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  919. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  920. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  921. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  922. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  923. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  924. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  925. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  926. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  927. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  928. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  929. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  930. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  931. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  932. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  933. "at java.lang.Thread.run(Thread.java:835) ~[?:?]"] }
  934. {"type": "server", "timestamp": "2019-07-01T18:49:22,335+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=59, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  935. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  936. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  937. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  938. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  939. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  940. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  941. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  942. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  943. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  944. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  945. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  946. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  947. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  948. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  949. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  950. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  951. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  952. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  953. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  954. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  955. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  956. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  957. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  958. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  959. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  960. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  961. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  962. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  963. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  964. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  965. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  966. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  967. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  968. "at java.lang.Thread.run(Thread.java:835) ~[?:?]"] }
  969. {"type": "server", "timestamp": "2019-07-01T18:49:22,650+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=60, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  970. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  971. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  972. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  973. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  974. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  975. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  976. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  977. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  978. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  979. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  980. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  981. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  982. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  983. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  984. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  985. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  986. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  987. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  988. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  989. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  990. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  991. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  992. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  993. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  994. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  995. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  996. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  997. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  998. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  999. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1000. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1001. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  1002. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  1003. "at java.lang.Thread.run(Thread.java:835) ~[?:?]"] }
  1004. {"type": "server", "timestamp": "2019-07-01T18:49:23,742+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=62, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  1005. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  1006. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  1007. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1008. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1009. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1010. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1011. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1012. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1013. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1014. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1015. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1016. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1017. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1018. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1019. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1020. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1021. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1022. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1023. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  1024. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  1025. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  1026. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  1027. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1028. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1029. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1030. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1031. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1032. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1033. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1034. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1035. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1036. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  1037. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  1038. "at java.lang.Thread.run(Thread.java:835) ~[?:?]"] }
  1039. {"type": "server", "timestamp": "2019-07-01T18:49:26,500+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=67, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  1040. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  1041. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  1042. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1043. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1044. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1045. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1046. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1047. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1048. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1049. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1050. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1051. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1052. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1053. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1054. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1055. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1056. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1057. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1058. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  1059. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  1060. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  1061. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  1062. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1063. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1064. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1065. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1066. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1067. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1068. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1069. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1070. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1071. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  1072. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  1073. "at java.lang.Thread.run(Thread.java:835) ~[?:?]"] }
  1074. {"type": "server", "timestamp": "2019-07-01T18:49:27,377+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=69, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-0}{9SdxPkMASPevUo7l0VZF8A}{YCNZJb5xQR-Qy7eOJTWtnA}{172.31.33.10}{172.31.33.10:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  1075. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.17.0.2:9300][internal:cluster/coordination/join]",
  1076. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  1077. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1078. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1079. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1080. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1081. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) ~[?:?]",
  1082. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1083. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) ~[?:?]",
  1084. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1085. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1086. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1087. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1088. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  1089. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  1090. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
  1091. {"type": "server", "timestamp": "2019-07-01T18:49:27,583+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=70, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-2}{xiuloUmzR3ipJJ-rw6P4Wg}{EUcKGpr1R3O2w0E1L4MK3g}{172.31.11.195}{172.31.11.195:9300}{ml.machine_memory=8061341696, ml.max_open_jobs=20, xpack.installed=true}}]}" ,
  1092. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-2][172.17.0.2:9300][internal:cluster/coordination/join]",
  1093. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  1094. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$3.onFailure(Coordinator.java:1350) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1095. "at org.elasticsearch.common.util.concurrent.ListenableFuture$1.run(ListenableFuture.java:101) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1096. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:192) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1097. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:92) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1098. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:54) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1099. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1290) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1100. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:124) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1101. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:172) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1102. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1103. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1104. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1105. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1106. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1107. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1108. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1109. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1110. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  1111. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  1112. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  1113. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: non-failed nodes do not form a quorum",
  1114. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCommitFailure(Publication.java:170) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1115. "at org.elasticsearch.cluster.coordination.Publication.access$600(Publication.java:41) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1116. "at org.elasticsearch.cluster.coordination.Publication$PublicationTarget$PublishResponseHandler.onFailure(Publication.java:348) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1117. "at org.elasticsearch.cluster.coordination.Coordinator$6.onFailure(Coordinator.java:1077) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1118. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$sendClusterStateToNode$6(PublicationTransportHandler.java:267) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1119. "at org.elasticsearch.cluster.coordination.PublicationTransportHandler$3.handleException(PublicationTransportHandler.java:285) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1120. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1118) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1121. "at org.elasticsearch.transport.TcpTransport.lambda$handleException$24(TcpTransport.java:1001) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1122. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  1123. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  1124. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  1125. "at java.lang.Thread.run(Thread.java:835) ~[?:?]"] }
  1126. {"type": "server", "timestamp": "2019-07-01T18:49:28,051+0000", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "clevyr-elk-cluster", "node.name": "elasticsearch-master-1", "cluster.uuid": "TKbXx1dXQYGdys3lqRf73Q", "node.id": "oKYbqreMTH6SMgX9xNlR3g",  "message": "failed to join {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20} with JoinRequest{sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=71, lastAcceptedTerm=35, lastAcceptedVersion=28, sourceNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}, targetNode={elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}}]}" ,
  1127. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [elasticsearch-master-1][172.31.29.199:9300][internal:cluster/coordination/join]",
  1128. "Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: received a newer join from {elasticsearch-master-1}{oKYbqreMTH6SMgX9xNlR3g}{PX_ntNucS8OR569_8sAVfg}{172.31.29.199}{172.31.29.199:9300}{ml.machine_memory=8061341696, xpack.installed=true, ml.max_open_jobs=20}",
  1129. "at org.elasticsearch.cluster.coordination.JoinHelper$CandidateJoinAccumulator.handleJoinRequest(JoinHelper.java:443) [elasticsearch-7.0.0.jar:7.0.0]",
  1130. "at org.elasticsearch.cluster.coordination.Coordinator.processJoinRequest(Coordinator.java:513) [elasticsearch-7.0.0.jar:7.0.0]",
  1131. "at org.elasticsearch.cluster.coordination.Coordinator.handleJoinRequest(Coordinator.java:478) [elasticsearch-7.0.0.jar:7.0.0]",
  1132. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$0(JoinHelper.java:124) [elasticsearch-7.0.0.jar:7.0.0]",
  1133. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:251) [x-pack-security-7.0.0.jar:7.0.0]",
  1134. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  1135. "at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:309) [x-pack-security-7.0.0.jar:7.0.0]",
  1136. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) [elasticsearch-7.0.0.jar:7.0.0]",
  1137. "at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:687) [elasticsearch-7.0.0.jar:7.0.0]",
  1138. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:751) [elasticsearch-7.0.0.jar:7.0.0]",
  1139. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-7.0.0.jar:7.0.0]",
  1140. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  1141. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  1142. "at java.lang.Thread.run(Thread.java:835) [?:?]"] }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!