API tools faq
paste
Advertisement
01n12

Elasticsearch LOgs

01n12
Jul 2nd, 2018
450
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 32.33 KB | None | 0 0
raw download clone embed print report
  1. ][o.e.n.Node ] [esmaster-node1] starting ...
  2. [2018-07-02T19:10:11,113][INFO ][o.e.t.TransportService ] [esmaster-node1] publish_address {172.23.13.207:8535}, bound_addresses {172.23.13.207:8535}
  3. [2018-07-02T19:10:11,194][INFO ][o.e.b.BootstrapChecks ] [esmaster-node1] bound or publishing to a non-loopback address, enforcing bootstrap checks
  4. [2018-07-02T19:10:14,375][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] detected_master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}, added {{escoord-node1}{vJ77jZGPQ26ZC5xJhDO-0g}{DqlqKURCRraXR3gW0Q3hOg}{172.23.13.210}{172.23.13.210:8545}{ml.machine_memory=6084067328, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},{esmaster-node2}{IchwVWtySY6SZjgR85sFcA}{ic6VmlGrSeeFIF3Pz2aOmA}{172.23.13.208}{172.23.13.208:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},{esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [12]])
  5. [2018-07-02T19:10:14,388][INFO ][o.e.c.s.ClusterSettings ] [esmaster-node1] updating [xpack.monitoring.collection.enabled] from [false] to [true]
  6. [2018-07-02T19:10:14,588][WARN ][o.e.x.s.a.s.m.NativeRoleMappingStore] [esmaster-node1] Failed to clear cache for realms [[]]
  7. [2018-07-02T19:10:14,589][INFO ][o.e.x.s.a.TokenService ] [esmaster-node1] refresh keys
  8. [2018-07-02T19:10:14,844][INFO ][o.e.x.s.a.TokenService ] [esmaster-node1] refreshed keys
  9. [2018-07-02T19:10:14,851][INFO ][o.e.l.LicenseService ] [esmaster-node1] license [18ce7e0c-c6e5-4f59-8dca-162d38fa2250] mode [basic] - valid
  10. [2018-07-02T19:10:14,866][INFO ][o.e.x.s.t.n.SecurityNetty4HttpServerTransport] [esmaster-node1] publish_address {172.23.13.207:8530}, bound_addresses {172.23.13.207:8530}
  11. [2018-07-02T19:10:14,867][INFO ][o.e.n.Node ] [esmaster-node1] started
  12. [2018-07-02T19:10:31,327][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] removed {{escoord-node1}{vJ77jZGPQ26ZC5xJhDO-0g}{DqlqKURCRraXR3gW0Q3hOg}{172.23.13.210}{172.23.13.210:8545}{ml.machine_memory=6084067328, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [13]])
  13. [2018-07-02T19:11:03,686][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] added {{escoord-node1}{vJ77jZGPQ26ZC5xJhDO-0g}{kvZlSXupRkqm2aaBczofhQ}{172.23.13.210}{172.23.13.210:8545}{ml.machine_memory=6084067328, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [14]])
  14. [2018-07-02T19:11:15,252][WARN ][o.e.x.m.e.l.LocalExporter] unexpected error while indexing monitoring document
  15. org.elasticsearch.xpack.monitoring.exporter.ExportException: UnavailableShardsException[[.monitoring-beats-6-2018.07.02][0] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[.monitoring-beats-6-2018.07.02][0]] containing [2] requests]]
  16. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$throwExportException$2(LocalBulk.java:128) ~[?:?]
  17. at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) ~[?:1.8.0_171]
  18. at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175) ~[?:1.8.0_171]
  19. at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:1.8.0_171]
  20. at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) ~[?:1.8.0_171]
  21. at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) ~[?:1.8.0_171]
  22. at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151) ~[?:1.8.0_171]
  23. at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174) ~[?:1.8.0_171]
  24. at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:1.8.0_171]
  25. at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418) ~[?:1.8.0_171]
  26. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.throwExportException(LocalBulk.java:129) ~[?:?]
  27. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$doFlush$0(LocalBulk.java:111) ~[?:?]
  28. at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.3.0.jar:6.3.0]
  29. at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-6.3.0.jar:6.3.0]
  30. at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:85) ~[elasticsearch-6.3.0.jar:6.3.0]
  31. at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:81) ~[elasticsearch-6.3.0.jar:6.3.0]
  32. at org.elasticsearch.action.bulk.TransportBulkAction$BulkRequestModifier.lambda$wrapActionListenerIfNeeded$0(TransportBulkAction.java:570) ~[elasticsearch-6.3.0.jar:6.3.0]
  33. at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) [elasticsearch-6.3.0.jar:6.3.0]
  34. at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:379) [elasticsearch-6.3.0.jar:6.3.0]
  35. at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:374) [elasticsearch-6.3.0.jar:6.3.0]
  36. at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:91) [elasticsearch-6.3.0.jar:6.3.0]
  37. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:897) [elasticsearch-6.3.0.jar:6.3.0]
  38. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:869) [elasticsearch-6.3.0.jar:6.3.0]
  39. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryBecauseUnavailable(TransportReplicationAction.java:928) [elasticsearch-6.3.0.jar:6.3.0]
  40. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryIfUnavailable(TransportReplicationAction.java:774) [elasticsearch-6.3.0.jar:6.3.0]
  41. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:727) [elasticsearch-6.3.0.jar:6.3.0]
  42. at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.3.0.jar:6.3.0]
  43. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:888) [elasticsearch-6.3.0.jar:6.3.0]
  44. at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:317) [elasticsearch-6.3.0.jar:6.3.0]
  45. at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:244) [elasticsearch-6.3.0.jar:6.3.0]
  46. at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:576) [elasticsearch-6.3.0.jar:6.3.0]
  47. at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:625) [elasticsearch-6.3.0.jar:6.3.0]
  48. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_171]
  49. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_171]
  50. at java.lang.Thread.run(Thread.java:748) [?:1.8.0_171]
  51. Caused by: org.elasticsearch.action.UnavailableShardsException: [.monitoring-beats-6-2018.07.02][0] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[.monitoring-beats-6-2018.07.02][0]] containing [2] requests]
  52. ... 12 more
  53. [2018-07-02T19:11:15,258][WARN ][o.e.x.m.e.l.LocalExporter] unexpected error while indexing monitoring document
  54. org.elasticsearch.xpack.monitoring.exporter.ExportException: UnavailableShardsException[[.monitoring-beats-6-2018.07.02][0] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[.monitoring-beats-6-2018.07.02][0]] containing [2] requests]]
  55. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$throwExportException$2(LocalBulk.java:128) ~[?:?]
  56. at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) ~[?:1.8.0_171]
  57. at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175) ~[?:1.8.0_171]
  58. at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:1.8.0_171]
  59. at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) ~[?:1.8.0_171]
  60. at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) ~[?:1.8.0_171]
  61. at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151) ~[?:1.8.0_171]
  62. at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174) ~[?:1.8.0_171]
  63. at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:1.8.0_171]
  64. at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418) ~[?:1.8.0_171]
  65. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.throwExportException(LocalBulk.java:129) ~[?:?]
  66. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$doFlush$0(LocalBulk.java:111) ~[?:?]
  67. at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.3.0.jar:6.3.0]
  68. at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-6.3.0.jar:6.3.0]
  69. at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:85) ~[elasticsearch-6.3.0.jar:6.3.0]
  70. at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:81) ~[elasticsearch-6.3.0.jar:6.3.0]
  71. at org.elasticsearch.action.bulk.TransportBulkAction$BulkRequestModifier.lambda$wrapActionListenerIfNeeded$0(TransportBulkAction.java:570) ~[elasticsearch-6.3.0.jar:6.3.0]
  72. at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) [elasticsearch-6.3.0.jar:6.3.0]
  73. at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:379) [elasticsearch-6.3.0.jar:6.3.0]
  74. at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:374) [elasticsearch-6.3.0.jar:6.3.0]
  75. at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:91) [elasticsearch-6.3.0.jar:6.3.0]
  76. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:897) [elasticsearch-6.3.0.jar:6.3.0]
  77. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:869) [elasticsearch-6.3.0.jar:6.3.0]
  78. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryBecauseUnavailable(TransportReplicationAction.java:928) [elasticsearch-6.3.0.jar:6.3.0]
  79. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryIfUnavailable(TransportReplicationAction.java:774) [elasticsearch-6.3.0.jar:6.3.0]
  80. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:727) [elasticsearch-6.3.0.jar:6.3.0]
  81. at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.3.0.jar:6.3.0]
  82. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:888) [elasticsearch-6.3.0.jar:6.3.0]
  83. at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:317) [elasticsearch-6.3.0.jar:6.3.0]
  84. at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:244) [elasticsearch-6.3.0.jar:6.3.0]
  85. at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:576) [elasticsearch-6.3.0.jar:6.3.0]
  86. at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:625) [elasticsearch-6.3.0.jar:6.3.0]
  87. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_171]
  88. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_171]
  89. at java.lang.Thread.run(Thread.java:748) [?:1.8.0_171]
  90. Caused by: org.elasticsearch.action.UnavailableShardsException: [.monitoring-beats-6-2018.07.02][0] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[.monitoring-beats-6-2018.07.02][0]] containing [2] requests]
  91. ... 12 more
  92. [2018-07-02T19:11:24,414][WARN ][o.e.x.m.e.l.LocalExporter] unexpected error while indexing monitoring document
  93. org.elasticsearch.xpack.monitoring.exporter.ExportException: UnavailableShardsException[[.monitoring-es-6-2018.07.02][0] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[.monitoring-es-6-2018.07.02][0]] containing [index {[.monitoring-es-6-2018.07.02][doc][fBNnXGQBHYg3LKBAMqSc], source[{"cluster_uuid":"RKSU_fYIQjKj2G2LqOUoHQ","timestamp":"2018-07-02T19:10:24.393Z","interval_ms":10000,"type":"node_stats","source_node":{"uuid":"oiuTlWkpT-O1KnlhxIfWGg","host":"172.23.13.207","transport_address":"172.23.13.207:8535","ip":"172.23.13.207","name":"esmaster-node1","timestamp":"2018-07-02T19:10:24.390Z"},"node_stats":{"node_id":"oiuTlWkpT-O1KnlhxIfWGg","node_master":false,"mlockall":true,"indices":{"docs":{"count":0},"store":{"size_in_bytes":0},"indexing":{"index_total":0,"index_time_in_millis":0,"throttle_time_in_millis":0},"search":{"query_total":0,"query_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"hit_count":0,"miss_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0}},"os":{"cpu":{"load_average":{"1m":0.51,"5m":0.21,"15m":0.12}},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":36191914201152},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}},"memory":{"control_group":"/","limit_in_bytes":"9223372036854771712","usage_in_bytes":"6531194880"}}},"process":{"open_file_descriptors":313,"max_file_descriptors":65536,"cpu":{"percent":15}},"jvm":{"mem":{"heap_used_in_bytes":269909080,"heap_used_percent":6,"heap_max_in_bytes":4260102144},"gc":{"collectors":{"young":{"collection_count":7,"collection_time_in_millis":369},"old":{"collection_count":2,"collection_time_in_millis":99}}}},"thread_pool":{"generic":{"threads":4,"queue":0,"rejected":0},"get":{"threads":0,"queue":0,"rejected":0},"index":{"threads":0,"queue":0,"rejected":0},"management":{"threads":2,"queue":0,"rejected":0},"search":{"threads":0,"queue":0,"rejected":0},"watcher":{"threads":0,"queue":0,"rejected":0},"write":{"threads":1,"queue":0,"rejected":0}},"fs":{"total":{"total_in_bytes":422621650944,"free_in_bytes":422211796992,"available_in_bytes":401772556288},"io_stats":{"total":{"operations":120,"read_operations":0,"write_operations":120,"read_kilobytes":0,"write_kilobytes":536}}}}}]}]]]
  94. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$throwExportException$2(LocalBulk.java:128) ~[?:?]
  95. at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193) ~[?:1.8.0_171]
  96. at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175) ~[?:1.8.0_171]
  97. at java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) ~[?:1.8.0_171]
  98. at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481) ~[?:1.8.0_171]
  99. at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471) ~[?:1.8.0_171]
  100. at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151) ~[?:1.8.0_171]
  101. at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174) ~[?:1.8.0_171]
  102. at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:1.8.0_171]
  103. at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418) ~[?:1.8.0_171]
  104. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.throwExportException(LocalBulk.java:129) ~[?:?]
  105. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$doFlush$0(LocalBulk.java:111) ~[?:?]
  106. at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.3.0.jar:6.3.0]
  107. at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-6.3.0.jar:6.3.0]
  108. at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:85) ~[elasticsearch-6.3.0.jar:6.3.0]
  109. at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:81) ~[elasticsearch-6.3.0.jar:6.3.0]
  110. at org.elasticsearch.action.bulk.TransportBulkAction$BulkRequestModifier.lambda$wrapActionListenerIfNeeded$0(TransportBulkAction.java:570) ~[elasticsearch-6.3.0.jar:6.3.0]
  111. at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) [elasticsearch-6.3.0.jar:6.3.0]
  112. at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:379) [elasticsearch-6.3.0.jar:6.3.0]
  113. at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:374) [elasticsearch-6.3.0.jar:6.3.0]
  114. at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:91) [elasticsearch-6.3.0.jar:6.3.0]
  115. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:897) [elasticsearch-6.3.0.jar:6.3.0]
  116. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:869) [elasticsearch-6.3.0.jar:6.3.0]
  117. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryBecauseUnavailable(TransportReplicationAction.java:928) [elasticsearch-6.3.0.jar:6.3.0]
  118. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryIfUnavailable(TransportReplicationAction.java:774) [elasticsearch-6.3.0.jar:6.3.0]
  119. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:727) [elasticsearch-6.3.0.jar:6.3.0]
  120. at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.3.0.jar:6.3.0]
  121. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:888) [elasticsearch-6.3.0.jar:6.3.0]
  122. at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:317) [elasticsearch-6.3.0.jar:6.3.0]
  123. at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:244) [elasticsearch-6.3.0.jar:6.3.0]
  124. at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:576) [elasticsearch-6.3.0.jar:6.3.0]
  125. at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:625) [elasticsearch-6.3.0.jar:6.3.0]
  126. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_171]
  127. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_171]
  128. at java.lang.Thread.run(Thread.java:748) [?:1.8.0_171]
  129. Caused by: org.elasticsearch.action.UnavailableShardsException: [.monitoring-es-6-2018.07.02][0] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[.monitoring-es-6-2018.07.02][0]] containing [index {[.monitoring-es-6-2018.07.02][doc][fBNnXGQBHYg3LKBAMqSc], source[{"cluster_uuid":"RKSU_fYIQjKj2G2LqOUoHQ","timestamp":"2018-07-02T19:10:24.393Z","interval_ms":10000,"type":"node_stats","source_node":{"uuid":"oiuTlWkpT-O1KnlhxIfWGg","host":"172.23.13.207","transport_address":"172.23.13.207:8535","ip":"172.23.13.207","name":"esmaster-node1","timestamp":"2018-07-02T19:10:24.390Z"},"node_stats":{"node_id":"oiuTlWkpT-O1KnlhxIfWGg","node_master":false,"mlockall":true,"indices":{"docs":{"count":0},"store":{"size_in_bytes":0},"indexing":{"index_total":0,"index_time_in_millis":0,"throttle_time_in_millis":0},"search":{"query_total":0,"query_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"hit_count":0,"miss_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0}},"os":{"cpu":{"load_average":{"1m":0.51,"5m":0.21,"15m":0.12}},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":36191914201152},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}},"memory":{"control_group":"/","limit_in_bytes":"9223372036854771712","usage_in_bytes":"6531194880"}}},"process":{"open_file_descriptors":313,"max_file_descriptors":65536,"cpu":{"percent":15}},"jvm":{"mem":{"heap_used_in_bytes":269909080,"heap_used_percent":6,"heap_max_in_bytes":4260102144},"gc":{"collectors":{"young":{"collection_count":7,"collection_time_in_millis":369},"old":{"collection_count":2,"collection_time_in_millis":99}}}},"thread_pool":{"generic":{"threads":4,"queue":0,"rejected":0},"get":{"threads":0,"queue":0,"rejected":0},"index":{"threads":0,"queue":0,"rejected":0},"management":{"threads":2,"queue":0,"rejected":0},"search":{"threads":0,"queue":0,"rejected":0},"watcher":{"threads":0,"queue":0,"rejected":0},"write":{"threads":1,"queue":0,"rejected":0}},"fs":{"total":{"total_in_bytes":422621650944,"free_in_bytes":422211796992,"available_in_bytes":401772556288},"io_stats":{"total":{"operations":120,"read_operations":0,"write_operations":120,"read_kilobytes":0,"write_kilobytes":536}}}}}]}]]
  130. ... 12 more
  131. [2018-07-02T19:11:24,417][WARN ][o.e.x.m.MonitoringService] [esmaster-node1] monitoring execution failed
  132. org.elasticsearch.xpack.monitoring.exporter.ExportException: Exception when closing export bulk
  133. at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1$1.<init>(ExportBulk.java:95) ~[?:?]
  134. at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1.onFailure(ExportBulk.java:93) ~[?:?]
  135. at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:206) ~[?:?]
  136. at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:200) ~[?:?]
  137. at org.elasticsearch.xpack.core.common.IteratingActionListener.onResponse(IteratingActionListener.java:96) ~[?:?]
  138. at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound.lambda$doFlush$0(ExportBulk.java:164) ~[?:?]
  139. at org.elasticsearch.action.ActionListener$1.onFailure(ActionListener.java:68) ~[elasticsearch-6.3.0.jar:6.3.0]
  140. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.throwExportException(LocalBulk.java:135) ~[?:?]
  141. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.lambda$doFlush$0(LocalBulk.java:111) ~[?:?]
  142. at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) ~[elasticsearch-6.3.0.jar:6.3.0]
  143. at org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) ~[elasticsearch-6.3.0.jar:6.3.0]
  144. at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:85) ~[elasticsearch-6.3.0.jar:6.3.0]
  145. at org.elasticsearch.action.support.TransportAction$1.onResponse(TransportAction.java:81) ~[elasticsearch-6.3.0.jar:6.3.0]
  146. at org.elasticsearch.action.bulk.TransportBulkAction$BulkRequestModifier.lambda$wrapActionListenerIfNeeded$0(TransportBulkAction.java:570) ~[elasticsearch-6.3.0.jar:6.3.0]
  147. at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:60) [elasticsearch-6.3.0.jar:6.3.0]
  148. at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.finishHim(TransportBulkAction.java:379) [elasticsearch-6.3.0.jar:6.3.0]
  149. at org.elasticsearch.action.bulk.TransportBulkAction$BulkOperation$1.onFailure(TransportBulkAction.java:374) [elasticsearch-6.3.0.jar:6.3.0]
  150. at org.elasticsearch.action.support.TransportAction$1.onFailure(TransportAction.java:91) [elasticsearch-6.3.0.jar:6.3.0]
  151. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.finishAsFailed(TransportReplicationAction.java:897) [elasticsearch-6.3.0.jar:6.3.0]
  152. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retry(TransportReplicationAction.java:869) [elasticsearch-6.3.0.jar:6.3.0]
  153. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryBecauseUnavailable(TransportReplicationAction.java:928) [elasticsearch-6.3.0.jar:6.3.0]
  154. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.retryIfUnavailable(TransportReplicationAction.java:774) [elasticsearch-6.3.0.jar:6.3.0]
  155. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase.doRun(TransportReplicationAction.java:727) [elasticsearch-6.3.0.jar:6.3.0]
  156. at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.3.0.jar:6.3.0]
  157. at org.elasticsearch.action.support.replication.TransportReplicationAction$ReroutePhase$2.onTimeout(TransportReplicationAction.java:888) [elasticsearch-6.3.0.jar:6.3.0]
  158. at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:317) [elasticsearch-6.3.0.jar:6.3.0]
  159. at org.elasticsearch.cluster.ClusterStateObserver$ObserverClusterStateListener.onTimeout(ClusterStateObserver.java:244) [elasticsearch-6.3.0.jar:6.3.0]
  160. at org.elasticsearch.cluster.service.ClusterApplierService$NotifyTimeout.run(ClusterApplierService.java:576) [elasticsearch-6.3.0.jar:6.3.0]
  161. at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:625) [elasticsearch-6.3.0.jar:6.3.0]
  162. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_171]
  163. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_171]
  164. at java.lang.Thread.run(Thread.java:748) [?:1.8.0_171]
  165. Caused by: org.elasticsearch.xpack.monitoring.exporter.ExportException: failed to flush export bulks
  166. at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound.lambda$doFlush$0(ExportBulk.java:156) ~[?:?]
  167. ... 26 more
  168. Caused by: org.elasticsearch.xpack.monitoring.exporter.ExportException: bulk [default_local] reports failures when exporting documents
  169. at org.elasticsearch.xpack.monitoring.exporter.local.LocalBulk.throwExportException(LocalBulk.java:124) ~[?:?]
  170. ... 24 more
  171. [2018-07-02T19:12:14,425][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] added {{esdatanode-1}{u4aTxQ4-RSSqsZOfpGMUYg}{AreGXsHDTBqKzs7pLdIFwQ}{172.23.13.205}{172.23.13.205:8515}{ml.machine_memory=14539759616, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [15]])
  172. [2018-07-02T19:12:58,849][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] added {{esdata-node2}{CBWtS2MERo-IJD2f_LgXIg}{ZuDatLqvS56fJKq-TA3bCA}{172.23.13.206}{172.23.13.206:8515}{ml.machine_memory=14539743232, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [20]])
  173. [2018-07-02T19:21:37,545][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] removed {{escoord-node1}{vJ77jZGPQ26ZC5xJhDO-0g}{kvZlSXupRkqm2aaBczofhQ}{172.23.13.210}{172.23.13.210:8545}{ml.machine_memory=6084067328, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [27]])
  174. [2018-07-02T19:21:58,421][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] added {{escoord-node1}{vJ77jZGPQ26ZC5xJhDO-0g}{R2OCgjDVTUyiReJ04iRDXA}{172.23.13.210}{172.23.13.210:8545}{ml.machine_memory=6084067328, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [28]])
  175. [2018-07-02T19:35:35,498][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] removed {{esdata-node2}{CBWtS2MERo-IJD2f_LgXIg}{ZuDatLqvS56fJKq-TA3bCA}{172.23.13.206}{172.23.13.206:8515}{ml.machine_memory=14539743232, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [34]])
  176. [2018-07-02T19:36:06,924][INFO ][o.e.c.s.ClusterApplierService] [esmaster-node1] added {{esdata-node2}{CBWtS2MERo-IJD2f_LgXIg}{HPQg2xmlT5uuQzrqyrH3rg}{172.23.13.206}{172.23.13.206:8515}{ml.machine_memory=14539743232, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true},}, reason: apply cluster state (from master [master {esmaster-node3}{gHb-Lt5LTRiIKs0gJZjokA}{cYLJRt4TTg25dLSuYcraug}{172.23.13.209}{172.23.13.209:8535}{ml.machine_memory=8197996544, rack=wag1-dc, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true} committed version [38]])
  177. $
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!