Advertisement
fidegnon1

ELKmessages3

Mar 29th, 2022
9
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 174.90 KB | None | 0 0
  1. Mar 22 18:55:49 XXXXXXXXX systemd: Started Kibana.
  2. Mar 22 18:55:54 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:54Z","tags":["plugin","warning"],"pid":7629,"path":"/usr/share/kibana/src/legacy/core_plugins/ems_util","message":"Skipping non-plugin directory at /usr/share/kibana/src/legacy/core_plugins/ems_util"}
  3. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:kibana@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  4. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:elasticsearch@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  5. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:xpack_main@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  6. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:graph@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  7. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:monitoring@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  8. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:spaces@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  9. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["security","warning"],"pid":7629,"message":"Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in kibana.yml"}
  10. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["security","warning"],"pid":7629,"message":"Session cookies will be transmitted over insecure connections. This is not recommended."}
  11. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:security@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  12. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:searchprofiler@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  13. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:ml@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  14. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:tilemap@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  15. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:watcher@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  16. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:grokdebugger@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  17. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:dashboard_mode@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  18. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:logstash@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  19. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:beats_management@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  20. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:apm@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  21. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:interpreter@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  22. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:canvas@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  23. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:license_management@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  24. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:index_management@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  25. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:console@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  26. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:console_extensions@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  27. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:notifications@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  28. Mar 22 18:55:55 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:55Z","tags":["status","plugin:index_lifecycle_management@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  29. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["status","plugin:infra@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  30. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["status","plugin:rollup@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  31. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["status","plugin:remote_clusters@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  32. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["status","plugin:cross_cluster_replication@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  33. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["status","plugin:upgrade_assistant@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  34. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["status","plugin:metrics@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  35. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["status","plugin:timelion@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  36. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["reporting","warning"],"pid":7629,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent pending reports from failing on restart, please set xpack.reporting.encryptionKey in kibana.yml"}
  37. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["status","plugin:reporting@6.6.2","info"],"pid":7629,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  38. Mar 22 18:55:56 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:56Z","tags":["reporting","browser-driver","warning"],"pid":7629,"message":"Enabling the Chromium sandbox provides an additional layer of protection."}
  39. Mar 22 18:55:58 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:55:58Z","tags":["status","plugin:elasticsearch@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  40. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["license","info","xpack"],"pid":7629,"message":"Imported license information from Elasticsearch for the [data] cluster: mode: basic | status: active"}
  41. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:xpack_main@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  42. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:graph@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  43. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:searchprofiler@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  44. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:ml@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  45. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:tilemap@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  46. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:watcher@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  47. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:grokdebugger@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  48. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:logstash@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  49. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:beats_management@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  50. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:index_management@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  51. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:index_lifecycle_management@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  52. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:rollup@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  53. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:remote_clusters@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  54. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:cross_cluster_replication@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  55. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:reporting@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  56. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["info","monitoring-ui","kibana-monitoring"],"pid":7629,"message":"Starting monitoring stats collection"}
  57. Mar 22 18:56:04 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:04Z","tags":["status","plugin:security@6.6.2","info"],"pid":7629,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  58. Mar 22 18:56:05 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:05Z","tags":["warning","stats-collection"],"pid":7629,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"term\\\":{\\\"type\\\":\\\"config\\\"}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  59. Mar 22 18:56:05 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:05Z","tags":["warning","stats-collection"],"pid":7629,"message":"Unable to fetch data from kql collector"}
  60. Mar 22 18:56:05 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:05Z","tags":["warning","stats-collection"],"pid":7629,"level":"error","error":{"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]","name":"Error","stack":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]] :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\"},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]"}
  61. Mar 22 18:56:05 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:05Z","tags":["warning","stats-collection"],"pid":7629,"message":"Unable to fetch data from kibana_settings collector"}
  62. Mar 22 18:56:05 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:05Z","tags":["warning","stats-collection"],"pid":7629,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  63. Mar 22 18:56:05 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:05Z","tags":["warning","stats-collection"],"pid":7629,"message":"Unable to fetch data from rollups collector"}
  64. Mar 22 18:56:05 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:05Z","tags":["warning","stats-collection"],"pid":7629,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  65. Mar 22 18:56:05 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:05Z","tags":["warning","stats-collection"],"pid":7629,"message":"Unable to fetch data from kibana collector"}
  66. Mar 22 18:56:06 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:06Z","tags":["warning","stats-collection"],"pid":7629,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  67. Mar 22 18:56:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:06Z","tags":["warning","stats-collection"],"pid":7629,"message":"Unable to fetch data from canvas collector"}
  68. Mar 22 18:56:08 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:08Z","tags":["license","info","xpack"],"pid":7629,"message":"Imported license information from Elasticsearch for the [monitoring] cluster: mode: basic | status: active"}
  69. Mar 22 18:56:15 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:15Z","tags":["warning","stats-collection"],"pid":7629,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  70. Mar 22 18:56:15 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:15Z","tags":["warning","stats-collection"],"pid":7629,"message":"Unable to fetch data from kibana collector"}
  71. Mar 22 18:56:15 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:15Z","tags":["warning","stats-collection"],"pid":7629,"level":"error","error":{"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]","name":"Error","stack":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]] :: {\"path\":\"/.kibana/doc/kql-telemetry%3Akql-telemetry\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]\\\"},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]"}
  72. Mar 22 18:56:15 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:15Z","tags":["warning","stats-collection"],"pid":7629,"message":"Unable to fetch data from kql collector"}
  73. Mar 22 18:56:15 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:15Z","tags":["status","plugin:spaces@6.6.2","error"],"pid":7629,"state":"red","message":"Status changed from yellow to red - all shards failed: [search_phase_execution_exception] all shards failed","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  74. Mar 22 18:56:15 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:15Z","tags":["fatal","root"],"pid":7629,"message":"{ [search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/doc/_count\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"should\\\":[{\\\"bool\\\":{\\\"must\\\":[{\\\"exists\\\":{\\\"field\\\":\\\"index-pattern\\\"}},{\\\"bool\\\":{\\\"must_not\\\":{\\\"term\\\":{\\\"migrationVersion.index-pattern\\\":\\\"6.5.0\\\"}}}}]}}]}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)\n status: 503,\n displayName: 'ServiceUnavailable',\n message:\n 'all shards failed: [search_phase_execution_exception] all shards failed',\n path: '/.kibana/doc/_count',\n query: {},\n body:\n { error:\n { root_cause: [],\n type: 'search_phase_execution_exception',\n reason: 'all shards failed',\n phase: 'query',\n grouped: true,\n failed_shards: [] },\n status: 503 },\n statusCode: 503,\n response:\n '{\"error\":{\"root_cause\":[],\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":[]},\"status\":503}',\n toString: [Function],\n toJSON: [Function],\n isBoom: true,\n isServer: true,\n data: null,\n output:\n { statusCode: 503,\n payload:\n { message:\n 'all shards failed: [search_phase_execution_exception] all shards failed',\n statusCode: 503,\n error: 'Service Unavailable' },\n headers: {} },\n reformat: [Function],\n [Symbol(SavedObjectsClientErrorCode)]: 'SavedObjectsClient/esUnavailable' }"}
  75. Mar 22 18:56:15 XXXXXXXXX kibana: FATAL [search_phase_execution_exception] all shards failed :: {"path":"/.kibana/doc/_count","query":{},"body":"{\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":[{\"exists\":{\"field\":\"index-pattern\"}},{\"bool\":{\"must_not\":{\"term\":{\"migrationVersion.index-pattern\":\"6.5.0\"}}}}]}}]}}}","statusCode":503,"response":"{\"error\":{\"root_cause\":[],\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":[]},\"status\":503}"}
  76. Mar 22 18:56:15 XXXXXXXXX systemd: kibana.service: main process exited, code=exited, status=1/FAILURE
  77. Mar 22 18:56:15 XXXXXXXXX systemd: Unit kibana.service entered failed state.
  78. Mar 22 18:56:15 XXXXXXXXX systemd: kibana.service failed.
  79. Mar 22 18:56:15 XXXXXXXXX systemd: kibana.service holdoff time over, scheduling restart.
  80. Mar 22 18:56:15 XXXXXXXXX systemd: Stopped Kibana.
  81. Mar 22 18:56:15 XXXXXXXXX systemd: Started Kibana.
  82. Mar 22 18:56:20 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:20Z","tags":["plugin","warning"],"pid":7646,"path":"/usr/share/kibana/src/legacy/core_plugins/ems_util","message":"Skipping non-plugin directory at /usr/share/kibana/src/legacy/core_plugins/ems_util"}
  83. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:kibana@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  84. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:elasticsearch@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  85. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:xpack_main@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  86. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:graph@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  87. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:monitoring@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  88. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:spaces@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  89. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["security","warning"],"pid":7646,"message":"Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in kibana.yml"}
  90. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["security","warning"],"pid":7646,"message":"Session cookies will be transmitted over insecure connections. This is not recommended."}
  91. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:security@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  92. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:searchprofiler@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  93. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:ml@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  94. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:tilemap@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  95. Mar 22 18:56:21 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:21Z","tags":["status","plugin:watcher@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  96. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:grokdebugger@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  97. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:dashboard_mode@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  98. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:logstash@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  99. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:beats_management@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  100. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:apm@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  101. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:interpreter@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  102. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:canvas@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  103. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:license_management@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  104. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:index_management@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  105. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:console@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  106. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:console_extensions@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  107. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:notifications@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  108. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:index_lifecycle_management@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  109. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:infra@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  110. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:rollup@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  111. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:remote_clusters@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  112. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:cross_cluster_replication@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  113. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:upgrade_assistant@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  114. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:metrics@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  115. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:timelion@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  116. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["reporting","warning"],"pid":7646,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent pending reports from failing on restart, please set xpack.reporting.encryptionKey in kibana.yml"}
  117. Mar 22 18:56:22 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:22Z","tags":["status","plugin:reporting@6.6.2","info"],"pid":7646,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  118. Mar 22 18:56:23 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:23Z","tags":["reporting","browser-driver","warning"],"pid":7646,"message":"Enabling the Chromium sandbox provides an additional layer of protection."}
  119. Mar 22 18:56:25 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:25Z","tags":["status","plugin:elasticsearch@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  120. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["license","info","xpack"],"pid":7646,"message":"Imported license information from Elasticsearch for the [data] cluster: mode: basic | status: active"}
  121. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:xpack_main@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  122. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:graph@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  123. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:searchprofiler@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  124. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:ml@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  125. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:tilemap@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  126. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:watcher@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  127. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:grokdebugger@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  128. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:logstash@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  129. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:beats_management@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  130. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:index_management@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  131. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:index_lifecycle_management@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  132. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:rollup@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  133. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:remote_clusters@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  134. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:cross_cluster_replication@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  135. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:reporting@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  136. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["info","monitoring-ui","kibana-monitoring"],"pid":7646,"message":"Starting monitoring stats collection"}
  137. Mar 22 18:56:30 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:30Z","tags":["status","plugin:security@6.6.2","info"],"pid":7646,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  138. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  139. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from canvas collector"}
  140. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  141. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from rollups collector"}
  142. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"term\\\":{\\\"type\\\":\\\"config\\\"}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  143. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kql collector"}
  144. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]","name":"Error","stack":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]] :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\"},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]"}
  145. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kibana_settings collector"}
  146. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  147. Mar 22 18:56:31 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:31Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kibana collector"}
  148. Mar 22 18:56:35 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:35Z","tags":["license","info","xpack"],"pid":7646,"message":"Imported license information from Elasticsearch for the [monitoring] cluster: mode: basic | status: active"}
  149. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  150. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kibana collector"}
  151. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"term\\\":{\\\"type\\\":\\\"config\\\"}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  152. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kql collector"}
  153. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  154. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from rollups collector"}
  155. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  156. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from canvas collector"}
  157. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]","name":"Error","stack":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]] :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\"},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]"}
  158. Mar 22 18:56:40 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:40Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kibana_settings collector"}
  159. Mar 22 18:56:51 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:51Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"term\\\":{\\\"type\\\":\\\"config\\\"}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  160. Mar 22 18:56:51 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:51Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kql collector"}
  161. Mar 22 18:56:51 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:51Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  162. Mar 22 18:56:51 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:51Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kibana collector"}
  163. Mar 22 18:56:53 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:53Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  164. Mar 22 18:56:53 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:53Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from rollups collector"}
  165. Mar 22 18:56:53 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:53Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  166. Mar 22 18:56:53 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:53Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from canvas collector"}
  167. Mar 22 18:56:54 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:56:54Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]","name":"Error","stack":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]] :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\"},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]"}
  168. Mar 22 18:56:54 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:56:54Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kibana_settings collector"}
  169. Mar 22 18:57:00 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:00Z","tags":["warning","stats-collection"],"pid":7646,"level":"error","error":{"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]","name":"Error","stack":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]] :: {\"path\":\"/.kibana/doc/kql-telemetry%3Akql-telemetry\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]\\\"},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[no_shard_available_action_exception] No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]"}
  170. Mar 22 18:57:00 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:00Z","tags":["warning","stats-collection"],"pid":7646,"message":"Unable to fetch data from kql collector"}
  171. Mar 22 18:57:01 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:01Z","tags":["status","plugin:spaces@6.6.2","error"],"pid":7646,"state":"red","message":"Status changed from yellow to red - all shards failed: [search_phase_execution_exception] all shards failed","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  172. Mar 22 18:57:01 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:01Z","tags":["fatal","root"],"pid":7646,"message":"{ [search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/doc/_count\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"should\\\":[{\\\"bool\\\":{\\\"must\\\":[{\\\"exists\\\":{\\\"field\\\":\\\"index-pattern\\\"}},{\\\"bool\\\":{\\\"must_not\\\":{\\\"term\\\":{\\\"migrationVersion.index-pattern\\\":\\\"6.5.0\\\"}}}}]}}]}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)\n status: 503,\n displayName: 'ServiceUnavailable',\n message:\n 'all shards failed: [search_phase_execution_exception] all shards failed',\n path: '/.kibana/doc/_count',\n query: {},\n body:\n { error:\n { root_cause: [],\n type: 'search_phase_execution_exception',\n reason: 'all shards failed',\n phase: 'query',\n grouped: true,\n failed_shards: [] },\n status: 503 },\n statusCode: 503,\n response:\n '{\"error\":{\"root_cause\":[],\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":[]},\"status\":503}',\n toString: [Function],\n toJSON: [Function],\n isBoom: true,\n isServer: true,\n data: null,\n output:\n { statusCode: 503,\n payload:\n { message:\n 'all shards failed: [search_phase_execution_exception] all shards failed',\n statusCode: 503,\n error: 'Service Unavailable' },\n headers: {} },\n reformat: [Function],\n [Symbol(SavedObjectsClientErrorCode)]: 'SavedObjectsClient/esUnavailable' }"}
  173. Mar 22 18:57:01 XXXXXXXXX kibana: FATAL [search_phase_execution_exception] all shards failed :: {"path":"/.kibana/doc/_count","query":{},"body":"{\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":[{\"exists\":{\"field\":\"index-pattern\"}},{\"bool\":{\"must_not\":{\"term\":{\"migrationVersion.index-pattern\":\"6.5.0\"}}}}]}}]}}}","statusCode":503,"response":"{\"error\":{\"root_cause\":[],\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"query\",\"grouped\":true,\"failed_shards\":[]},\"status\":503}"}
  174. Mar 22 18:57:01 XXXXXXXXX systemd: kibana.service: main process exited, code=exited, status=1/FAILURE
  175. Mar 22 18:57:01 XXXXXXXXX systemd: Unit kibana.service entered failed state.
  176. Mar 22 18:57:01 XXXXXXXXX systemd: kibana.service failed.
  177. Mar 22 18:57:01 XXXXXXXXX systemd: kibana.service holdoff time over, scheduling restart.
  178. Mar 22 18:57:01 XXXXXXXXX systemd: Stopped Kibana.
  179. Mar 22 18:57:01 XXXXXXXXX systemd: Started Kibana.
  180. Mar 22 18:57:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:06Z","tags":["plugin","warning"],"pid":7658,"path":"/usr/share/kibana/src/legacy/core_plugins/ems_util","message":"Skipping non-plugin directory at /usr/share/kibana/src/legacy/core_plugins/ems_util"}
  181. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:kibana@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  182. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:elasticsearch@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  183. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:xpack_main@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  184. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:graph@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  185. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:monitoring@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  186. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:spaces@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  187. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["security","warning"],"pid":7658,"message":"Generating a random key for xpack.security.encryptionKey. To prevent sessions from being invalidated on restart, please set xpack.security.encryptionKey in kibana.yml"}
  188. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["security","warning"],"pid":7658,"message":"Session cookies will be transmitted over insecure connections. This is not recommended."}
  189. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:security@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  190. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:searchprofiler@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  191. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:ml@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  192. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:tilemap@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  193. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:watcher@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  194. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:grokdebugger@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  195. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:dashboard_mode@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  196. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:logstash@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  197. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:beats_management@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  198. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:apm@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  199. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:interpreter@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  200. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:canvas@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  201. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:license_management@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  202. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:index_management@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  203. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:console@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  204. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:console_extensions@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  205. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:notifications@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  206. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:index_lifecycle_management@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  207. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:infra@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  208. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:rollup@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  209. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:remote_clusters@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  210. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:cross_cluster_replication@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  211. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:upgrade_assistant@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  212. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:metrics@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  213. Mar 22 18:57:07 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:07Z","tags":["status","plugin:timelion@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
  214. Mar 22 18:57:08 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:08Z","tags":["reporting","warning"],"pid":7658,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent pending reports from failing on restart, please set xpack.reporting.encryptionKey in kibana.yml"}
  215. Mar 22 18:57:08 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:08Z","tags":["status","plugin:reporting@6.6.2","info"],"pid":7658,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
  216. Mar 22 18:57:08 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:08Z","tags":["reporting","browser-driver","warning"],"pid":7658,"message":"Enabling the Chromium sandbox provides an additional layer of protection."}
  217. Mar 22 18:57:09 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:09Z","tags":["status","plugin:elasticsearch@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  218. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["license","info","xpack"],"pid":7658,"message":"Imported license information from Elasticsearch for the [data] cluster: mode: basic | status: active"}
  219. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:xpack_main@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  220. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:graph@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  221. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:searchprofiler@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  222. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:ml@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  223. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:tilemap@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  224. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:watcher@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  225. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:grokdebugger@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  226. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:logstash@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  227. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:beats_management@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  228. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:index_management@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  229. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:index_lifecycle_management@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  230. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:rollup@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  231. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:remote_clusters@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  232. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:cross_cluster_replication@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  233. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:reporting@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  234. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["info","monitoring-ui","kibana-monitoring"],"pid":7658,"message":"Starting monitoring stats collection"}
  235. Mar 22 18:57:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:12Z","tags":["status","plugin:security@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from yellow to green - Ready","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  236. Mar 22 18:57:16 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:16Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  237. Mar 22 18:57:16 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:16Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana collector"}
  238. Mar 22 18:57:16 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:16Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  239. Mar 22 18:57:16 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:16Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from rollups collector"}
  240. Mar 22 18:57:16 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:16Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"term\\\":{\\\"type\\\":\\\"config\\\"}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  241. Mar 22 18:57:16 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:16Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kql collector"}
  242. Mar 22 18:57:17 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:17Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  243. Mar 22 18:57:17 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:17Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from canvas collector"}
  244. Mar 22 18:57:17 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:17Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  245. Mar 22 18:57:17 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:17Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana_settings collector"}
  246. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"term\\\":{\\\"type\\\":\\\"config\\\"}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  247. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kql collector"}
  248. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  249. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from rollups collector"}
  250. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  251. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana_settings collector"}
  252. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  253. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana collector"}
  254. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  255. Mar 22 18:57:23 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:23Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from canvas collector"}
  256. Mar 22 18:57:24 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:24Z","tags":["license","info","xpack"],"pid":7658,"message":"Imported license information from Elasticsearch for the [monitoring] cluster: mode: basic | status: active"}
  257. Mar 22 18:57:37 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:37Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  258. Mar 22 18:57:37 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:37Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana collector"}
  259. Mar 22 18:57:38 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:38Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/kql-telemetry%3Akql-telemetry\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  260. Mar 22 18:57:38 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:38Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kql collector"}
  261. Mar 22 18:57:41 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:41Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  262. Mar 22 18:57:41 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:41Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana_settings collector"}
  263. Mar 22 18:57:43 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:43Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{},\"body\":\"{\\\"query\\\":{\\\"term\\\":{\\\"type\\\":\\\"config\\\"}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  264. Mar 22 18:57:43 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:43Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kql collector"}
  265. Mar 22 18:57:43 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:43Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  266. Mar 22 18:57:43 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:43Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from canvas collector"}
  267. Mar 22 18:57:44 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:44Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  268. Mar 22 18:57:44 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:44Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from rollups collector"}
  269. Mar 22 18:57:45 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:45Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  270. Mar 22 18:57:45 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:45Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from canvas collector"}
  271. Mar 22 18:57:45 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:45Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  272. Mar 22 18:57:45 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:45Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana_settings collector"}
  273. Mar 22 18:57:46 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:46Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  274. Mar 22 18:57:46 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:46Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from rollups collector"}
  275. Mar 22 18:57:50 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:50Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  276. Mar 22 18:57:50 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:50Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana collector"}
  277. Mar 22 18:57:53 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:53Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  278. Mar 22 18:57:53 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:53Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from canvas collector"}
  279. Mar 22 18:57:53 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:53Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  280. Mar 22 18:57:53 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:53Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana collector"}
  281. Mar 22 18:57:59 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:57:59Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  282. Mar 22 18:57:59 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:57:59Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from rollups collector"}
  283. Mar 22 18:58:02 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:02Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/kql-telemetry%3Akql-telemetry\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  284. Mar 22 18:58:02 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:02Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kql collector"}
  285. Mar 22 18:58:05 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:05Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  286. Mar 22 18:58:05 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:05Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana_settings collector"}
  287. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:xpack_main@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  288. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:graph@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  289. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:spaces@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from yellow to red - Request Timeout after 3000ms","prevState":"yellow","prevMsg":"Waiting for Elasticsearch"}
  290. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:security@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  291. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:searchprofiler@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  292. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:ml@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  293. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:tilemap@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  294. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:watcher@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  295. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:grokdebugger@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  296. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:logstash@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  297. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:beats_management@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  298. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:index_management@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  299. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:index_lifecycle_management@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  300. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:rollup@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  301. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:remote_clusters@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  302. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:cross_cluster_replication@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  303. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:reporting@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  304. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["status","plugin:elasticsearch@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from green to red - Request Timeout after 3000ms","prevState":"green","prevMsg":"Ready"}
  305. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:06Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/kql-telemetry%3Akql-telemetry\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  306. Mar 22 18:58:06 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:06Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kql collector"}
  307. Mar 22 18:58:09 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:09Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  308. Mar 22 18:58:09 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:09Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from rollups collector"}
  309. Mar 22 18:58:10 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:10Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  310. Mar 22 18:58:10 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:10Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana_settings collector"}
  311. Mar 22 18:58:11 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:11Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  312. Mar 22 18:58:11 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:11Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from canvas collector"}
  313. Mar 22 18:58:12 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:12Z","tags":["license","warning","xpack"],"pid":7658,"message":"License information from the X-Pack plugin could not be obtained from Elasticsearch for the [data] cluster. Error: Request Timeout after 30000ms"}
  314. Mar 22 18:58:13 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:13Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/kql-telemetry%3Akql-telemetry\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][kql-telemetry:kql-telemetry]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  315. Mar 22 18:58:13 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:13Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kql collector"}
  316. Mar 22 18:58:13 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:13Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"Request Timeout after 30000ms","name":"Error","stack":"Error: Request Timeout after 30000ms\n at /usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:355:15\n at Timeout.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:384:7)\n at ontimeout (timers.js:436:11)\n at tryOnTimeout (timers.js:300:5)\n at listOnTimeout (timers.js:263:5)\n at Timer.processTimers (timers.js:223:10)"},"message":"Request Timeout after 30000ms"}
  317. Mar 22 18:58:13 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:13Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from reporting collector"}
  318. Mar 22 18:58:14 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:14Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"ignore_unavailable\":true,\"filter_path\":\"aggregations.types.buckets\"},\"body\":\"{\\\"size\\\":0,\\\"query\\\":{\\\"terms\\\":{\\\"type\\\":[\\\"dashboard\\\",\\\"visualization\\\",\\\"search\\\",\\\"index-pattern\\\",\\\"graph-workspace\\\",\\\"timelion-sheet\\\"]}},\\\"aggs\\\":{\\\"types\\\":{\\\"terms\\\":{\\\"field\\\":\\\"type\\\",\\\"size\\\":6}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  319. Mar 22 18:58:14 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:14Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana collector"}
  320. Mar 22 18:58:14 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:14Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":10000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._source.canvas-workpad\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"type\\\":\\\"canvas-workpad\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  321. Mar 22 18:58:14 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:14Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from canvas collector"}
  322. Mar 22 18:58:14 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:14Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[search_phase_execution_exception] all shards failed","name":"Error","stack":"[search_phase_execution_exception] all shards failed :: {\"path\":\"/.kibana/_search\",\"query\":{\"size\":1000,\"ignore_unavailable\":true,\"filter_path\":\"hits.hits._id\"},\"body\":\"{\\\"query\\\":{\\\"bool\\\":{\\\"filter\\\":{\\\"term\\\":{\\\"index-pattern.type\\\":\\\"rollup\\\"}}}}}\",\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[],\\\"type\\\":\\\"search_phase_execution_exception\\\",\\\"reason\\\":\\\"all shards failed\\\",\\\"phase\\\":\\\"query\\\",\\\"grouped\\\":true,\\\"failed_shards\\\":[]},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[search_phase_execution_exception] all shards failed"}
  323. Mar 22 18:58:14 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:14Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from rollups collector"}
  324. Mar 22 18:58:14 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:14Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }","name":"Error","stack":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" } :: {\"path\":\"/.kibana/doc/config%3A6.6.2\",\"query\":{},\"statusCode\":503,\"response\":\"{\\\"error\\\":{\\\"root_cause\\\":[{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}],\\\"type\\\":\\\"no_shard_available_action_exception\\\",\\\"reason\\\":\\\"No shard available for [get [.kibana][doc][config:6.6.2]: routing [null]]\\\",\\\"caused_by\\\":{\\\"type\\\":\\\"illegal_index_shard_state_exception\\\",\\\"reason\\\":\\\"CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED]\\\",\\\"index_uuid\\\":\\\"IPFXUZtwTKupDzIvGIlQmg\\\",\\\"shard\\\":\\\"0\\\",\\\"index\\\":\\\".kibana_1\\\"}},\\\"status\\\":503}\"}\n at respond (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:308:15)\n at checkRespForFailure (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:267:7)\n at HttpConnector.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/connectors/http.js:166:7)\n at IncomingMessage.wrapper (/usr/share/kibana/node_modules/elasticsearch/node_modules/lodash/lodash.js:4935:19)\n at IncomingMessage.emit (events.js:194:15)\n at endReadableNT (_stream_readable.js:1103:12)\n at process._tickCallback (internal/process/next_tick.js:63:19)"},"message":"[illegal_index_shard_state_exception] CurrentState[RECOVERING] operations only allowed when shard state is one of [POST_RECOVERY, STARTED], with { index_uuid=\"IPFXUZtwTKupDzIvGIlQmg\" & shard=\"0\" & index=\".kibana_1\" }"}
  325. Mar 22 18:58:14 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:14Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana_settings collector"}
  326. Mar 22 18:58:16 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:16Z","tags":["status","plugin:elasticsearch@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  327. Mar 22 18:58:25 XXXXXXXXX logstash: [2022-03-22T18:58:25,795][WARN ][logstash.outputs.elasticsearch] Marking url as dead. Last error: [LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError] Elasticsearch Unreachable: [http://localhost:9200/][Manticore::SocketTimeout] Read timed out {:url=>http://localhost:9200/, :error_message=>"Elasticsearch Unreachable: [http://localhost:9200/][Manticore::SocketTimeout] Read timed out", :error_class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError"}
  328. Mar 22 18:58:25 XXXXXXXXX logstash: [2022-03-22T18:58:25,813][ERROR][logstash.outputs.elasticsearch] Attempted to send a bulk request to elasticsearch' but Elasticsearch appears to be unreachable or down! {:error_message=>"Elasticsearch Unreachable: [http://localhost:9200/][Manticore::SocketTimeout] Read timed out", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError", :will_retry_in_seconds=>2}
  329. Mar 22 18:58:27 XXXXXXXXX logstash: [2022-03-22T18:58:27,852][ERROR][logstash.outputs.elasticsearch] Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down? {:error_message=>"No Available connections", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError", :will_retry_in_seconds=>4}
  330. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["license","info","xpack"],"pid":7658,"message":"Imported license information from Elasticsearch for the [data] cluster: mode: basic | status: active"}
  331. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:xpack_main@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  332. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:graph@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  333. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:searchprofiler@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  334. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:ml@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  335. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:tilemap@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  336. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:watcher@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  337. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:grokdebugger@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  338. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:logstash@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  339. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:beats_management@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  340. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:index_management@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  341. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:index_lifecycle_management@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  342. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:rollup@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  343. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:remote_clusters@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  344. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:cross_cluster_replication@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  345. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:reporting@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  346. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["info","monitoring-ui","kibana-monitoring"],"pid":7658,"message":"Starting monitoring stats collection"}
  347. Mar 22 18:58:29 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:29Z","tags":["status","plugin:security@6.6.2","info"],"pid":7658,"state":"green","message":"Status changed from red to green - Ready","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  348. Mar 22 18:58:31 XXXXXXXXX logstash: [2022-03-22T18:58:31,078][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://localhost:9200/"}
  349. Mar 22 18:58:31 XXXXXXXXX logstash: [2022-03-22T18:58:31,860][ERROR][logstash.outputs.elasticsearch] Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down? {:error_message=>"No Available connections", :class=>"LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError", :will_retry_in_seconds=>8}
  350. Mar 22 18:58:42 XXXXXXXXX kibana: {"type":"error","@timestamp":"2022-03-22T17:58:42Z","tags":["warning","stats-collection"],"pid":7658,"level":"error","error":{"message":"Request Timeout after 30000ms","name":"Error","stack":"Error: Request Timeout after 30000ms\n at /usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:355:15\n at Timeout.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:384:7)\n at ontimeout (timers.js:436:11)\n at tryOnTimeout (timers.js:300:5)\n at listOnTimeout (timers.js:263:5)\n at Timer.processTimers (timers.js:223:10)"},"message":"Request Timeout after 30000ms"}
  351. Mar 22 18:58:42 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:42Z","tags":["warning","stats-collection"],"pid":7658,"message":"Unable to fetch data from kibana collector"}
  352. Mar 22 18:58:43 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:43Z","tags":["status","plugin:spaces@6.6.2","error"],"pid":7658,"state":"red","message":"Status changed from red to red - Request Timeout after 30000ms","prevState":"red","prevMsg":"Request Timeout after 3000ms"}
  353. Mar 22 18:58:43 XXXXXXXXX kibana: {"type":"log","@timestamp":"2022-03-22T17:58:43Z","tags":["fatal","root"],"pid":7658,"message":"{ Error: Request Timeout after 30000ms\n at /usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:355:15\n at Timeout.<anonymous> (/usr/share/kibana/node_modules/elasticsearch/src/lib/transport.js:384:7)\n at ontimeout (timers.js:436:11)\n at tryOnTimeout (timers.js:300:5)\n at listOnTimeout (timers.js:263:5)\n at Timer.processTimers (timers.js:223:10)\n status: undefined,\n displayName: 'RequestTimeout',\n message: 'Request Timeout after 30000ms',\n body: undefined,\n isBoom: true,\n isServer: true,\n data: null,\n output:\n { statusCode: 503,\n payload:\n { statusCode: 503,\n error: 'Service Unavailable',\n message: 'Request Timeout after 30000ms' },\n headers: {} },\n reformat: [Function],\n [Symbol(SavedObjectsClientErrorCode)]: 'SavedObjectsClient/esUnavailable' }"}
  354. Mar 22 18:58:43 XXXXXXXXX kibana: FATAL Error: Request Timeout after 30000ms
  355. Mar 22 18:58:43 XXXXXXXXX systemd: kibana.service: main process exited, code=exited, status=1/FAILURE
  356. Mar 22 18:58:43 XXXXXXXXX systemd: Unit kibana.service entered failed state.
  357. Mar 22 18:58:43 XXXXXXXXX systemd: kibana.service failed.
  358. Mar 22 18:58:44 XXXXXXXXX systemd: kibana.service holdoff time over, scheduling restart.
  359. Mar 22 18:58:44 XXXXXXXXX systemd: Stopped Kibana.
  360.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement