Advertisement
nytramworc

Untitled

Oct 16th, 2019
284
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 56.40 KB | None | 0 0
  1. OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
  2. {"type": "server", "timestamp": "2019-10-16T10:27:10,156Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/sde)]], net usable_space [31.3gb], net total_space [31.3gb], types [ext4]" }
  3. {"type": "server", "timestamp": "2019-10-16T10:27:10,162Z", "level": "INFO", "component": "o.e.e.NodeEnvironment", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "heap size [1.9gb], compressed ordinary object pointers [true]" }
  4. {"type": "server", "timestamp": "2019-10-16T10:27:10,166Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "node name [d2c-es-cluster-nonprod-development-master-1], node ID [SzHXpWGaTdCbGcT4R-8jVg], cluster name [d2c-es-cluster-nonprod-development]" }
  5. {"type": "server", "timestamp": "2019-10-16T10:27:10,167Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "version[7.4.0], pid[1], build[default/docker/22e1767283e61a198cb4db791ea66e3f11ab9910/2019-09-27T08:36:48.569419Z], OS[Linux/4.15.0-1059-azure/amd64], JVM[AdoptOpenJDK/OpenJDK 64-Bit Server VM/13/13+33]" }
  6. {"type": "server", "timestamp": "2019-10-16T10:27:10,168Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "JVM home [/usr/share/elasticsearch/jdk]" }
  7. {"type": "server", "timestamp": "2019-10-16T10:27:10,168Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "JVM arguments [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dio.netty.allocator.numDirectArenas=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/tmp/elasticsearch-13811412419998825929, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.locale.providers=COMPAT, -Des.cgroups.hierarchy.override=/, -Xmx2g, -Xms2g, -Dio.netty.allocator.type=pooled, -XX:MaxDirectMemorySize=1073741824, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker, -Des.bundled_jdk=true]" }
  8. {"type": "server", "timestamp": "2019-10-16T10:27:12,807Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [aggs-matrix-stats]" }
  9. {"type": "server", "timestamp": "2019-10-16T10:27:12,807Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [analysis-common]" }
  10. {"type": "server", "timestamp": "2019-10-16T10:27:12,808Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [data-frame]" }
  11. {"type": "server", "timestamp": "2019-10-16T10:27:12,808Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [flattened]" }
  12. {"type": "server", "timestamp": "2019-10-16T10:27:12,808Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [frozen-indices]" }
  13. {"type": "server", "timestamp": "2019-10-16T10:27:12,808Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [ingest-common]" }
  14. {"type": "server", "timestamp": "2019-10-16T10:27:12,809Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [ingest-geoip]" }
  15. {"type": "server", "timestamp": "2019-10-16T10:27:12,809Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [ingest-user-agent]" }
  16. {"type": "server", "timestamp": "2019-10-16T10:27:12,809Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [lang-expression]" }
  17. {"type": "server", "timestamp": "2019-10-16T10:27:12,809Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [lang-mustache]" }
  18. {"type": "server", "timestamp": "2019-10-16T10:27:12,810Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [lang-painless]" }
  19. {"type": "server", "timestamp": "2019-10-16T10:27:12,810Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [mapper-extras]" }
  20. {"type": "server", "timestamp": "2019-10-16T10:27:12,810Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [parent-join]" }
  21. {"type": "server", "timestamp": "2019-10-16T10:27:12,810Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [percolator]" }
  22. {"type": "server", "timestamp": "2019-10-16T10:27:12,810Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [rank-eval]" }
  23. {"type": "server", "timestamp": "2019-10-16T10:27:12,811Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [reindex]" }
  24. {"type": "server", "timestamp": "2019-10-16T10:27:12,811Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [repository-url]" }
  25. {"type": "server", "timestamp": "2019-10-16T10:27:12,811Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [search-business-rules]" }
  26. {"type": "server", "timestamp": "2019-10-16T10:27:12,811Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [spatial]" }
  27. {"type": "server", "timestamp": "2019-10-16T10:27:12,812Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [transport-netty4]" }
  28. {"type": "server", "timestamp": "2019-10-16T10:27:12,813Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [vectors]" }
  29. {"type": "server", "timestamp": "2019-10-16T10:27:12,813Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-analytics]" }
  30. {"type": "server", "timestamp": "2019-10-16T10:27:12,813Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-ccr]" }
  31. {"type": "server", "timestamp": "2019-10-16T10:27:12,813Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-core]" }
  32. {"type": "server", "timestamp": "2019-10-16T10:27:12,814Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-deprecation]" }
  33. {"type": "server", "timestamp": "2019-10-16T10:27:12,814Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-graph]" }
  34. {"type": "server", "timestamp": "2019-10-16T10:27:12,814Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-ilm]" }
  35. {"type": "server", "timestamp": "2019-10-16T10:27:12,814Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-logstash]" }
  36. {"type": "server", "timestamp": "2019-10-16T10:27:12,814Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-ml]" }
  37. {"type": "server", "timestamp": "2019-10-16T10:27:12,815Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-monitoring]" }
  38. {"type": "server", "timestamp": "2019-10-16T10:27:12,815Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-rollup]" }
  39. {"type": "server", "timestamp": "2019-10-16T10:27:12,815Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-security]" }
  40. {"type": "server", "timestamp": "2019-10-16T10:27:12,815Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-sql]" }
  41. {"type": "server", "timestamp": "2019-10-16T10:27:12,816Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-voting-only-node]" }
  42. {"type": "server", "timestamp": "2019-10-16T10:27:12,816Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "loaded module [x-pack-watcher]" }
  43. {"type": "server", "timestamp": "2019-10-16T10:27:12,816Z", "level": "INFO", "component": "o.e.p.PluginsService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "no plugins loaded" }
  44. {"type": "server", "timestamp": "2019-10-16T10:27:17,419Z", "level": "INFO", "component": "o.e.x.m.p.l.CppLogMessageHandler", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "[controller/86] [Main.cc@110] controller (64 bit): Version 7.4.0 (Build 11d694e7bae395) Copyright (c) 2019 Elasticsearch BV" }
  45. {"type": "server", "timestamp": "2019-10-16T10:27:18,464Z", "level": "INFO", "component": "o.e.d.DiscoveryModule", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "using discovery type [zen] and seed hosts providers [settings]" }
  46. {"type": "server", "timestamp": "2019-10-16T10:27:19,434Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "initialized" }
  47. {"type": "server", "timestamp": "2019-10-16T10:27:19,434Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "starting ..." }
  48. {"type": "server", "timestamp": "2019-10-16T10:27:19,590Z", "level": "INFO", "component": "o.e.t.TransportService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "publish_address {10.253.156.126:9300}, bound_addresses {0.0.0.0:9300}" }
  49. {"type": "server", "timestamp": "2019-10-16T10:27:19,600Z", "level": "INFO", "component": "o.e.b.BootstrapChecks", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "bound or publishing to a non-loopback address, enforcing bootstrap checks" }
  50. {"type": "server", "timestamp": "2019-10-16T10:27:26,193Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "setting initial configuration to VotingConfiguration{{bootstrap-placeholder}-d2c-es-cluster-nonprod-development-master-0,xe4u3BBWQ6aXDUXwksqNzg,SzHXpWGaTdCbGcT4R-8jVg}" }
  51. {"type": "server", "timestamp": "2019-10-16T10:27:26,436Z", "level": "INFO", "component": "o.e.c.c.CoordinationState", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "cluster UUID set to [tckw_lvBS7W4uynZeODAkw]" }
  52. {"type": "server", "timestamp": "2019-10-16T10:27:26,503Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "master node changed {previous [], current [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}]}, added {{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true},}, term: 1, version: 1, reason: ApplyCommitRequest{term=1, version=1, sourceNode={d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}}" }
  53. {"type": "server", "timestamp": "2019-10-16T10:27:26,543Z", "level": "INFO", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "publish_address {10.253.156.126:9200}, bound_addresses {0.0.0.0:9200}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  54. {"type": "server", "timestamp": "2019-10-16T10:27:26,544Z", "level": "INFO", "component": "o.e.n.Node", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "started", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  55. {"type": "server", "timestamp": "2019-10-16T10:27:26,793Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  56. {"type": "server", "timestamp": "2019-10-16T10:27:27,097Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  57. {"type": "server", "timestamp": "2019-10-16T10:27:27,193Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  58. {"type": "server", "timestamp": "2019-10-16T10:27:27,292Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  59. {"type": "server", "timestamp": "2019-10-16T10:27:27,445Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  60. {"type": "server", "timestamp": "2019-10-16T10:27:27,542Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  61. {"type": "server", "timestamp": "2019-10-16T10:27:27,654Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  62. {"type": "server", "timestamp": "2019-10-16T10:27:27,816Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  63. {"type": "server", "timestamp": "2019-10-16T10:27:27,910Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  64. {"type": "server", "timestamp": "2019-10-16T10:27:28,050Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  65. {"type": "server", "timestamp": "2019-10-16T10:27:28,209Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  66. {"type": "server", "timestamp": "2019-10-16T10:27:28,306Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  67. {"type": "server", "timestamp": "2019-10-16T10:27:28,881Z", "level": "INFO", "component": "o.e.l.LicenseService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "license [dd5de6cf-dfd3-4a87-9a65-81ad9501be9d] mode [basic] - valid", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  68. {"type": "server", "timestamp": "2019-10-16T10:27:28,886Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  69. {"type": "server", "timestamp": "2019-10-16T10:27:29,019Z", "level": "INFO", "component": "o.e.x.m.e.l.LocalExporter", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "waiting for elected master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] to setup local exporter [default_local] (does it have x-pack installed?)", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  70. {"type": "server", "timestamp": "2019-10-16T10:27:43,367Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "added {{d2c-es-cluster-nonprod-development-master-0}{ghCS57fITWyVAvzEUYYCLQ}{_QIra1z6TPiPt1gzA3zb_w}{10.253.157.3}{10.253.157.3:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true},}, term: 1, version: 17, reason: ApplyCommitRequest{term=1, version=17, sourceNode={d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  71. {"type": "server", "timestamp": "2019-10-16T12:39:20,809Z", "level": "DEBUG", "component": "o.e.a.a.i.s.g.TransportGetSettingsAction", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "connection exception while trying to forward request with action name [indices:monitor/settings/get] to master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}], scheduling a retry. Error: [org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][indices:monitor/settings/get] disconnected]", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  72. {"type": "server", "timestamp": "2019-10-16T12:39:20,812Z", "level": "DEBUG", "component": "o.e.a.a.i.s.g.TransportGetSettingsAction", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "timed out while retrying [indices:monitor/settings/get] after failure (timeout [30s])", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  73. "stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][indices:monitor/settings/get] disconnected"] }
  74. {"type": "server", "timestamp": "2019-10-16T12:39:20,810Z", "level": "INFO", "component": "o.e.c.c.Coordinator", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] failed, restarting discovery", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  75. "stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][disconnected] disconnected"] }
  76. {"type": "server", "timestamp": "2019-10-16T12:39:20,817Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "master node changed {previous [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}], current []}, term: 1, version: 18, reason: becoming candidate: onLeaderFailure", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  77. {"type": "server", "timestamp": "2019-10-16T12:39:20,815Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "path: /_cat/indices, params: {}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  78. "stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][indices:monitor/settings/get] disconnected]",
  79. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:214) [elasticsearch-7.4.0.jar:7.4.0]",
  80. "at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325) [elasticsearch-7.4.0.jar:7.4.0]",
  81. "at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:149) [elasticsearch-7.4.0.jar:7.4.0]",
  82. "at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:120) [elasticsearch-7.4.0.jar:7.4.0]",
  83. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:198) [elasticsearch-7.4.0.jar:7.4.0]",
  84. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$500(TransportMasterNodeAction.java:108) [elasticsearch-7.4.0.jar:7.4.0]",
  85. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$1.handleException(TransportMasterNodeAction.java:184) [elasticsearch-7.4.0.jar:7.4.0]",
  86. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1120) [elasticsearch-7.4.0.jar:7.4.0]",
  87. "at org.elasticsearch.transport.TransportService$8.run(TransportService.java:982) [elasticsearch-7.4.0.jar:7.4.0]",
  88. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) [elasticsearch-7.4.0.jar:7.4.0]",
  89. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  90. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  91. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  92. "Caused by: org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][indices:monitor/settings/get] disconnected"] }
  93. {"type": "server", "timestamp": "2019-10-16T12:39:20,825Z", "level": "DEBUG", "component": "o.e.a.a.c.h.TransportClusterHealthAction", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "connection exception while trying to forward request with action name [cluster:monitor/health] to master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}], scheduling a retry. Error: [org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:monitor/health] disconnected]", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  94. {"type": "server", "timestamp": "2019-10-16T12:39:20,830Z", "level": "DEBUG", "component": "o.e.a.a.c.h.TransportClusterHealthAction", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "timed out while retrying [cluster:monitor/health] after failure (timeout [30s])", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  95. "stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:monitor/health] disconnected"] }
  96. {"type": "server", "timestamp": "2019-10-16T12:39:20,831Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "path: /_cat/health, params: {}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  97. "stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:monitor/health] disconnected]",
  98. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:214) [elasticsearch-7.4.0.jar:7.4.0]",
  99. "at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325) [elasticsearch-7.4.0.jar:7.4.0]",
  100. "at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:149) [elasticsearch-7.4.0.jar:7.4.0]",
  101. "at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:120) [elasticsearch-7.4.0.jar:7.4.0]",
  102. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:198) [elasticsearch-7.4.0.jar:7.4.0]",
  103. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$500(TransportMasterNodeAction.java:108) [elasticsearch-7.4.0.jar:7.4.0]",
  104. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$1.handleException(TransportMasterNodeAction.java:184) [elasticsearch-7.4.0.jar:7.4.0]",
  105. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1120) [elasticsearch-7.4.0.jar:7.4.0]",
  106. "at org.elasticsearch.transport.TransportService$8.run(TransportService.java:982) [elasticsearch-7.4.0.jar:7.4.0]",
  107. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) [elasticsearch-7.4.0.jar:7.4.0]",
  108. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  109. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  110. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  111. "Caused by: org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:monitor/health] disconnected"] }
  112. {"type": "server", "timestamp": "2019-10-16T12:39:20,835Z", "level": "DEBUG", "component": "o.e.a.a.c.s.TransportPutStoredScriptAction", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "connection exception while trying to forward request with action name [cluster:admin/script/put] to master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}], scheduling a retry. Error: [org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:admin/script/put] disconnected]", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  113. {"type": "server", "timestamp": "2019-10-16T12:39:20,836Z", "level": "DEBUG", "component": "o.e.a.a.c.h.TransportClusterHealthAction", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "connection exception while trying to forward request with action name [cluster:monitor/health] to master node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}], scheduling a retry. Error: [org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:monitor/health] disconnected]", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  114. {"type": "server", "timestamp": "2019-10-16T12:39:20,836Z", "level": "DEBUG", "component": "o.e.a.a.c.h.TransportClusterHealthAction", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "timed out while retrying [cluster:monitor/health] after failure (timeout [30s])", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  115. "stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:monitor/health] disconnected"] }
  116. {"type": "server", "timestamp": "2019-10-16T12:39:20,837Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "path: /_cat/health, params: {}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  117. "stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:monitor/health] disconnected]",
  118. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:214) [elasticsearch-7.4.0.jar:7.4.0]",
  119. "at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325) [elasticsearch-7.4.0.jar:7.4.0]",
  120. "at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:149) [elasticsearch-7.4.0.jar:7.4.0]",
  121. "at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:120) [elasticsearch-7.4.0.jar:7.4.0]",
  122. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:198) [elasticsearch-7.4.0.jar:7.4.0]",
  123. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$500(TransportMasterNodeAction.java:108) [elasticsearch-7.4.0.jar:7.4.0]",
  124. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$1.handleException(TransportMasterNodeAction.java:184) [elasticsearch-7.4.0.jar:7.4.0]",
  125. "at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1120) [elasticsearch-7.4.0.jar:7.4.0]",
  126. "at org.elasticsearch.transport.TransportService$8.run(TransportService.java:982) [elasticsearch-7.4.0.jar:7.4.0]",
  127. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) [elasticsearch-7.4.0.jar:7.4.0]",
  128. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  129. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  130. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  131. "Caused by: org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:monitor/health] disconnected"] }
  132. {"type": "server", "timestamp": "2019-10-16T12:39:30,819Z", "level": "WARN", "component": "o.e.c.c.ClusterFormationFailureHelper", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "master not discovered or elected yet, an election requires at least 2 nodes with ids from [xe4u3BBWQ6aXDUXwksqNzg, ghCS57fITWyVAvzEUYYCLQ, SzHXpWGaTdCbGcT4R-8jVg], have discovered [{d2c-es-cluster-nonprod-development-master-1}{SzHXpWGaTdCbGcT4R-8jVg}{rbjvlsaTQV-Lb_P_NgdONg}{10.253.156.126}{10.253.156.126:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}, {d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}, {d2c-es-cluster-nonprod-development-master-0}{ghCS57fITWyVAvzEUYYCLQ}{_QIra1z6TPiPt1gzA3zb_w}{10.253.157.3}{10.253.157.3:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] which is a quorum; discovery will continue using [10.253.156.42:9300, 10.253.157.3:9300] from hosts providers and [{d2c-es-cluster-nonprod-development-master-1}{SzHXpWGaTdCbGcT4R-8jVg}{rbjvlsaTQV-Lb_P_NgdONg}{10.253.156.126}{10.253.156.126:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}, {d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}, {d2c-es-cluster-nonprod-development-master-0}{ghCS57fITWyVAvzEUYYCLQ}{_QIra1z6TPiPt1gzA3zb_w}{10.253.157.3}{10.253.157.3:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}] from last-known cluster state; node term 1, last-accepted version 18 in term 1", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
  133. {"type": "server", "timestamp": "2019-10-16T12:39:40,431Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "failed to join {d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={d2c-es-cluster-nonprod-development-master-1}{SzHXpWGaTdCbGcT4R-8jVg}{rbjvlsaTQV-Lb_P_NgdONg}{10.253.156.126}{10.253.156.126:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=1, lastAcceptedTerm=0, lastAcceptedVersion=0, sourceNode={d2c-es-cluster-nonprod-development-master-1}{SzHXpWGaTdCbGcT4R-8jVg}{rbjvlsaTQV-Lb_P_NgdONg}{10.253.156.126}{10.253.156.126:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}, targetNode={d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}}]}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  134. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][internal:cluster/coordination/join]",
  135. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: publication failed",
  136. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication$4.onFailure(Coordinator.java:1429) ~[elasticsearch-7.4.0.jar:7.4.0]",
  137. "at org.elasticsearch.action.ActionRunnable.onFailure(ActionRunnable.java:60) ~[elasticsearch-7.4.0.jar:7.4.0]",
  138. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:39) ~[elasticsearch-7.4.0.jar:7.4.0]",
  139. "at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.4.0.jar:7.4.0]",
  140. "at org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:93) ~[elasticsearch-7.4.0.jar:7.4.0]",
  141. "at org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:55) ~[elasticsearch-7.4.0.jar:7.4.0]",
  142. "at org.elasticsearch.cluster.coordination.Coordinator$CoordinatorPublication.onCompletion(Coordinator.java:1349) ~[elasticsearch-7.4.0.jar:7.4.0]",
  143. "at org.elasticsearch.cluster.coordination.Publication.onPossibleCompletion(Publication.java:125) ~[elasticsearch-7.4.0.jar:7.4.0]",
  144. "at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:89) ~[elasticsearch-7.4.0.jar:7.4.0]",
  145. "at org.elasticsearch.cluster.coordination.Coordinator.cancelActivePublication(Coordinator.java:1126) ~[elasticsearch-7.4.0.jar:7.4.0]",
  146. "at org.elasticsearch.cluster.coordination.Coordinator.becomeCandidate(Coordinator.java:541) ~[elasticsearch-7.4.0.jar:7.4.0]",
  147. "at org.elasticsearch.cluster.coordination.Coordinator.joinLeaderInTerm(Coordinator.java:456) ~[elasticsearch-7.4.0.jar:7.4.0]",
  148. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:136) ~[elasticsearch-7.4.0.jar:7.4.0]",
  149. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.4.0.jar:7.4.0]",
  150. "at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) ~[elasticsearch-7.4.0.jar:7.4.0]",
  151. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773) ~[elasticsearch-7.4.0.jar:7.4.0]",
  152. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.4.0.jar:7.4.0]",
  153. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  154. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  155. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  156. "Caused by: org.elasticsearch.ElasticsearchException: publication cancelled before committing: become candidate: joinLeaderInTerm",
  157. "at org.elasticsearch.cluster.coordination.Publication.cancel(Publication.java:86) ~[elasticsearch-7.4.0.jar:7.4.0]",
  158. "at org.elasticsearch.cluster.coordination.Coordinator.cancelActivePublication(Coordinator.java:1126) ~[elasticsearch-7.4.0.jar:7.4.0]",
  159. "at org.elasticsearch.cluster.coordination.Coordinator.becomeCandidate(Coordinator.java:541) ~[elasticsearch-7.4.0.jar:7.4.0]",
  160. "at org.elasticsearch.cluster.coordination.Coordinator.joinLeaderInTerm(Coordinator.java:456) ~[elasticsearch-7.4.0.jar:7.4.0]",
  161. "at org.elasticsearch.cluster.coordination.JoinHelper.lambda$new$2(JoinHelper.java:136) ~[elasticsearch-7.4.0.jar:7.4.0]",
  162. "at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.4.0.jar:7.4.0]",
  163. "at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) ~[elasticsearch-7.4.0.jar:7.4.0]",
  164. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:773) ~[elasticsearch-7.4.0.jar:7.4.0]",
  165. "at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.4.0.jar:7.4.0]",
  166. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  167. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  168. "at java.lang.Thread.run(Thread.java:830) ~[?:?]"] }
  169. {"type": "server", "timestamp": "2019-10-16T12:39:40,462Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "failed to join {d2c-es-cluster-nonprod-development-master-0}{ghCS57fITWyVAvzEUYYCLQ}{_QIra1z6TPiPt1gzA3zb_w}{10.253.157.3}{10.253.157.3:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={d2c-es-cluster-nonprod-development-master-1}{SzHXpWGaTdCbGcT4R-8jVg}{rbjvlsaTQV-Lb_P_NgdONg}{10.253.156.126}{10.253.156.126:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=2, lastAcceptedTerm=1, lastAcceptedVersion=18, sourceNode={d2c-es-cluster-nonprod-development-master-1}{SzHXpWGaTdCbGcT4R-8jVg}{rbjvlsaTQV-Lb_P_NgdONg}{10.253.156.126}{10.253.156.126:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}, targetNode={d2c-es-cluster-nonprod-development-master-0}{ghCS57fITWyVAvzEUYYCLQ}{_QIra1z6TPiPt1gzA3zb_w}{10.253.157.3}{10.253.157.3:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}}]}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  170. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [d2c-es-cluster-nonprod-development-master-0][10.253.157.3:9300][internal:cluster/coordination/join]",
  171. "Caused by: org.elasticsearch.cluster.coordination.FailedToCommitClusterStateException: node is no longer master for term 2 while handling publication",
  172. "at org.elasticsearch.cluster.coordination.Coordinator.publish(Coordinator.java:1049) ~[elasticsearch-7.4.0.jar:7.4.0]",
  173. "at org.elasticsearch.cluster.service.MasterService.publish(MasterService.java:268) ~[elasticsearch-7.4.0.jar:7.4.0]",
  174. "at org.elasticsearch.cluster.service.MasterService.runTasks(MasterService.java:250) ~[elasticsearch-7.4.0.jar:7.4.0]",
  175. "at org.elasticsearch.cluster.service.MasterService.access$000(MasterService.java:73) ~[elasticsearch-7.4.0.jar:7.4.0]",
  176. "at org.elasticsearch.cluster.service.MasterService$Batcher.run(MasterService.java:151) ~[elasticsearch-7.4.0.jar:7.4.0]",
  177. "at org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-7.4.0.jar:7.4.0]",
  178. "at org.elasticsearch.cluster.service.TaskBatcher$BatchedTask.run(TaskBatcher.java:188) ~[elasticsearch-7.4.0.jar:7.4.0]",
  179. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) ~[elasticsearch-7.4.0.jar:7.4.0]",
  180. "at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsThreadPoolExecutor.java:252) ~[elasticsearch-7.4.0.jar:7.4.0]",
  181. "at org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsThreadPoolExecutor.java:215) ~[elasticsearch-7.4.0.jar:7.4.0]",
  182. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]",
  183. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]",
  184. "at java.lang.Thread.run(Thread.java:830) [?:?]"] }
  185. {"type": "server", "timestamp": "2019-10-16T12:39:49,636Z", "level": "DEBUG", "component": "o.e.a.a.c.s.TransportPutStoredScriptAction", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "timed out while retrying [cluster:admin/script/put] after failure (timeout [30s])", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  186. "stacktrace": ["org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:admin/script/put] disconnected"] }
  187. {"type": "server", "timestamp": "2019-10-16T12:39:49,638Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "path: /_scripts/test, params: {id=test}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  188. "stacktrace": ["org.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:admin/script/put] disconnected]",
  189. "at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$2.onTimeout(TransportMasterNodeAction.java:214) [elasticsearch-7.4.0.jar:7.4.0]",
  190. "at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:325) [elasticsearch-7.4.0.jar:7.4.0]",
  191. "at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:252) [elasticsearch-7.4.0.jar:7.4.0]",
  192. "at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:598) [elasticsearch-7.4.0.jar:7.4.0]",
  193. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:703) [elasticsearch-7.4.0.jar:7.4.0]",
  194. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  195. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  196. "at java.lang.Thread.run(Thread.java:830) [?:?]",
  197. "Caused by: org.elasticsearch.transport.NodeDisconnectedException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][cluster:admin/script/put] disconnected"] }
  198. {"type": "server", "timestamp": "2019-10-16T12:40:01,951Z", "level": "INFO", "component": "o.e.c.c.JoinHelper", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "failed to join {d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true} with JoinRequest{sourceNode={d2c-es-cluster-nonprod-development-master-1}{SzHXpWGaTdCbGcT4R-8jVg}{rbjvlsaTQV-Lb_P_NgdONg}{10.253.156.126}{10.253.156.126:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}, optionalJoin=Optional[Join{term=3, lastAcceptedTerm=1, lastAcceptedVersion=18, sourceNode={d2c-es-cluster-nonprod-development-master-1}{SzHXpWGaTdCbGcT4R-8jVg}{rbjvlsaTQV-Lb_P_NgdONg}{10.253.156.126}{10.253.156.126:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}, targetNode={d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}}]}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" ,
  199. "stacktrace": ["org.elasticsearch.transport.RemoteTransportException: [d2c-es-cluster-nonprod-development-master-2][10.253.156.42:9300][internal:cluster/coordination/join]",
  200. "Caused by: org.elasticsearch.cluster.NotMasterException: Node [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, xpack.installed=true, ml.max_open_jobs=20}] not master for join request"] }
  201. {"type": "server", "timestamp": "2019-10-16T12:40:02,160Z", "level": "INFO", "component": "o.e.c.s.ClusterApplierService", "cluster.name": "d2c-es-cluster-nonprod-development", "node.name": "d2c-es-cluster-nonprod-development-master-1", "message": "master node changed {previous [], current [{d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}]}, term: 4, version: 21, reason: ApplyCommitRequest{term=4, version=21, sourceNode={d2c-es-cluster-nonprod-development-master-2}{xe4u3BBWQ6aXDUXwksqNzg}{I-jQhqecTT-ekXIjZytOOg}{10.253.156.42}{10.253.156.42:9300}{dilm}{ml.machine_memory=5368709120, ml.max_open_jobs=20, xpack.installed=true}}", "cluster.uuid": "tckw_lvBS7W4uynZeODAkw", "node.id": "SzHXpWGaTdCbGcT4R-8jVg" }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement