API tools faq
paste
Advertisement
rabidsloth

ECK death log

rabidsloth
Jul 28th, 2020
121
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
JSON 114.80 KB | None | 0 0
raw download clone embed print report
  1. W0728 08:38:37.080852       1 reflector.go:326] pkg/mod/k8s.io/client-go@v0.17.2/tools/cache/reflector.go:105: watch of *v1.StatefulSet ended with: too old resource version: 49757382 (51519444)
  2. {"log.level":"info","@timestamp":"2020-07-28T08:38:38.090Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":461,"namespace":"default","es_name":"clustername"}
  3. {"log.level":"info","@timestamp":"2020-07-28T08:38:38.784Z","log.logger":"zen2","message":"Ensuring no voting exclusions are set","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  4. {"log.level":"info","@timestamp":"2020-07-28T08:38:38.898Z","log.logger":"migrate-data","message":"Setting routing allocation excludes","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","value":"none_excluded"}
  5. {"log.level":"info","@timestamp":"2020-07-28T08:38:39.022Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":461,"namespace":"default","es_name":"clustername","took":0.931555165}
  6. W0728 08:38:48.956851       1 reflector.go:326] pkg/mod/k8s.io/client-go@v0.17.2/tools/cache/reflector.go:105: watch of *v1.Elasticsearch ended with: too old resource version: 51452889 (51519452)
  7. {"log.level":"info","@timestamp":"2020-07-28T08:38:49.958Z","log.logger":"license-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":397,"namespace":"default","es_name":"clustername"}
  8. {"log.level":"info","@timestamp":"2020-07-28T08:38:49.959Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":181,"namespace":"default","kibana_name":"ik-kibana"}
  9. {"log.level":"info","@timestamp":"2020-07-28T08:38:49.970Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":462,"namespace":"default","es_name":"clustername"}
  10. {"log.level":"info","@timestamp":"2020-07-28T08:38:49.989Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":253,"namespace":"default","es_name":"clustername"}
  11. {"log.level":"info","@timestamp":"2020-07-28T08:38:49.990Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":253,"namespace":"default","es_name":"clustername","took":0.000460976}
  12. {"log.level":"info","@timestamp":"2020-07-28T08:38:49.990Z","log.logger":"license-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":397,"namespace":"default","es_name":"clustername","took":0.031497373}
  13. {"log.level":"info","@timestamp":"2020-07-28T08:38:50.387Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":181,"namespace":"default","kibana_name":"ik-kibana","took":0.427445965}
  14. {"log.level":"info","@timestamp":"2020-07-28T08:38:50.614Z","log.logger":"zen2","message":"Ensuring no voting exclusions are set","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  15. {"log.level":"info","@timestamp":"2020-07-28T08:38:50.732Z","log.logger":"migrate-data","message":"Setting routing allocation excludes","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","value":"none_excluded"}
  16. {"log.level":"info","@timestamp":"2020-07-28T08:38:50.845Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":462,"namespace":"default","es_name":"clustername","took":0.875555754}
  17. {"log.level":"info","@timestamp":"2020-07-28T08:39:29.853Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":463,"namespace":"default","es_name":"clustername"}
  18. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.353Z","log.logger":"driver","message":"Creating master node","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","actualReplicas":0,"targetReplicas":1}
  19. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.353Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":0}
  20. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.354Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"StatefulSet","namespace":"default","name":"clustername-es-default"}
  21. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.386Z","log.logger":"migrate-data","message":"Setting routing allocation excludes","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","value":"none_excluded"}
  22. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.398Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":463,"namespace":"default","es_name":"clustername","took":0.54503486}
  23. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.398Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":464,"namespace":"default","es_name":"clustername"}
  24. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.424Z","log.logger":"license-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":398,"namespace":"default","es_name":"clustername"}
  25. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.425Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":182,"namespace":"default","kibana_name":"ik-kibana"}
  26. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.444Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":254,"namespace":"default","es_name":"clustername"}
  27. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.444Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":254,"namespace":"default","es_name":"clustername","took":0.000486496}
  28. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.464Z","log.logger":"license-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":398,"namespace":"default","es_name":"clustername","took":0.040083566}
  29. {"log.level":"info","@timestamp":"2020-07-28T08:39:30.855Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":182,"namespace":"default","kibana_name":"ik-kibana","took":0.430569027}
  30. {"log.level":"info","@timestamp":"2020-07-28T08:39:31.024Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":464,"namespace":"default","es_name":"clustername","took":0.626260843}
  31. {"log.level":"info","@timestamp":"2020-07-28T08:39:31.024Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":465,"namespace":"default","es_name":"clustername"}
  32. {"log.level":"info","@timestamp":"2020-07-28T08:39:31.573Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":465,"namespace":"default","es_name":"clustername","took":0.548591275}
  33. {"log.level":"info","@timestamp":"2020-07-28T08:39:35.899Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":466,"namespace":"default","es_name":"clustername"}
  34. {"log.level":"info","@timestamp":"2020-07-28T08:39:35.900Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"ConfigMap","namespace":"default","name":"clustername-es-scripts"}
  35. {"log.level":"info","@timestamp":"2020-07-28T08:39:35.901Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":255,"namespace":"default","es_name":"clustername"}
  36. {"log.level":"info","@timestamp":"2020-07-28T08:39:35.902Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":255,"namespace":"default","es_name":"clustername","took":0.000658579}
  37. {"log.level":"info","@timestamp":"2020-07-28T08:39:35.984Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":256,"namespace":"default","es_name":"clustername"}
  38. {"log.level":"info","@timestamp":"2020-07-28T08:39:35.985Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":256,"namespace":"default","es_name":"clustername","took":0.000862431}
  39. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.075Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":257,"namespace":"default","es_name":"clustername"}
  40. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.075Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":257,"namespace":"default","es_name":"clustername","took":0.000469006}
  41. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.278Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":466,"namespace":"default","es_name":"clustername","took":0.378987647}
  42. {"log.level":"error","@timestamp":"2020-07-28T08:39:36.278Z","log.logger":"controller-runtime.controller","message":"Reconciler error","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","controller":"elasticsearch-controller","request":"default/clustername","error":"Service "clustername-es-http" not found","errorCauses":[{"error":"Service "clustername-es-http" not found"}],"error.stack_trace":"github.com/go-logr/zapr.(*zapLogger).Error
  43.     /go/pkg/mod/github.com/go-logr/zapr@v0.1.0/zapr.go:128
  44. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  45.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:258
  46. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  47.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  48. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  49.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  50. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  51.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  52. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  53.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  54. k8s.io/apimachinery/pkg/util/wait.Until
  55.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88"}
  56. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.278Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":467,"namespace":"default","es_name":"clustername"}
  57. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.279Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Service","namespace":"default","name":"clustername-es-http"}
  58. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.324Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-remote-ca"}
  59. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.371Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":258,"namespace":"default","es_name":"clustername"}
  60. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.372Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":258,"namespace":"default","es_name":"clustername","took":0.000599458}
  61. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.589Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-elastic-user"}
  62. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.610Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":259,"namespace":"default","es_name":"clustername"}
  63. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.610Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":259,"namespace":"default","es_name":"clustername","took":0.000407046}
  64. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.731Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":260,"namespace":"default","es_name":"clustername"}
  65. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.731Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":260,"namespace":"default","es_name":"clustername","took":0.000580798}
  66. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.804Z","log.logger":"generic-reconciler","message":"Updating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-xpack-file-realm"}
  67. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.810Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":261,"namespace":"default","es_name":"clustername"}
  68. {"log.level":"info","@timestamp":"2020-07-28T08:39:36.810Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":261,"namespace":"default","es_name":"clustername","took":0.000568797}
  69. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.199Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":468,"namespace":"logging","es_name":"clustername"}
  70. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.199Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":468,"namespace":"logging","es_name":"clustername","took":0.000085701}
  71. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.199Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":262,"namespace":"logging","es_name":"clustername"}
  72. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.199Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":262,"namespace":"logging","es_name":"clustername","took":0.000216563}
  73. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.250Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":263,"namespace":"default","es_name":"clustername"}
  74. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.250Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":263,"namespace":"default","es_name":"clustername","took":0.000379225}
  75. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.299Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":264,"namespace":"default","es_name":"clustername"}
  76. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.299Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":264,"namespace":"default","es_name":"clustername","took":0.000363674}
  77. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.349Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":469,"namespace":"monitoring","es_name":"clustername"}
  78. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.349Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":469,"namespace":"monitoring","es_name":"clustername","took":0.000074321}
  79. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.349Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":265,"namespace":"monitoring","es_name":"clustername"}
  80. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.349Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":265,"namespace":"monitoring","es_name":"clustername","took":0.000177141}
  81. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.549Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":183,"namespace":"default","kibana_name":"ik-kibana"}
  82. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.577Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":266,"namespace":"default","es_name":"clustername"}
  83. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.578Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":266,"namespace":"default","es_name":"clustername","took":0.000379045}
  84. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.638Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":267,"namespace":"default","es_name":"clustername"}
  85. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.638Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":267,"namespace":"default","es_name":"clustername","took":0.000378195}
  86. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.649Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"default-ik-kibana-kibana-user"}
  87. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.650Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":268,"namespace":"default","es_name":"clustername"}
  88. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.650Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":268,"namespace":"default","es_name":"clustername","took":0.000408085}
  89. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.654Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":183,"namespace":"default","kibana_name":"ik-kibana","took":0.105057355}
  90. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.654Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":184,"namespace":"default","kibana_name":"ik-kibana"}
  91. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.668Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":269,"namespace":"default","es_name":"clustername"}
  92. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.668Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":269,"namespace":"default","es_name":"clustername","took":0.000323385}
  93. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.760Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":270,"namespace":"default","es_name":"clustername"}
  94. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.760Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":270,"namespace":"default","es_name":"clustername","took":0.000352474}
  95. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.856Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":184,"namespace":"default","kibana_name":"ik-kibana","took":0.201273679}
  96. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.900Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":271,"namespace":"default","es_name":"clustername"}
  97. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.901Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":271,"namespace":"default","es_name":"clustername","took":0.000533977}
  98. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.951Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":272,"namespace":"default","es_name":"clustername"}
  99. {"log.level":"info","@timestamp":"2020-07-28T08:39:37.952Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":272,"namespace":"default","es_name":"clustername","took":0.000345125}
  100. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.050Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":185,"namespace":"default","kibana_name":"ik-kibana"}
  101. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.076Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":273,"namespace":"default","es_name":"clustername"}
  102. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.076Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":273,"namespace":"default","es_name":"clustername","took":0.000464496}
  103. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.261Z","log.logger":"kibana-association-controller","message":"Updating Kibana spec with Elasticsearch backend configuration","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","kibana_name":"ik-kibana"}
  104. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.293Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":185,"namespace":"default","kibana_name":"ik-kibana","took":0.243099481}
  105. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.297Z","log.logger":"kibana-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":171,"namespace":"default","kibana_name":"ik-kibana"}
  106. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.297Z","log.logger":"association","message":"Elasticsearch association not established: skipping associated resource deployment reconciliation","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Kibana","namespace":"default","name":"ik-kibana"}
  107. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.297Z","log.logger":"kibana-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":171,"namespace":"default","kibana_name":"ik-kibana","took":0.000257573}
  108. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.297Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":186,"namespace":"default","kibana_name":"ik-kibana"}
  109. {"log.level":"info","@timestamp":"2020-07-28T08:39:38.519Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":186,"namespace":"default","kibana_name":"ik-kibana","took":0.222054253}
  110. {"log.level":"info","@timestamp":"2020-07-28T08:40:34.029Z","log.logger":"kibana-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":172,"namespace":"default","kibana_name":"ik-kibana"}
  111. {"log.level":"info","@timestamp":"2020-07-28T08:40:34.029Z","log.logger":"association","message":"Elasticsearch association not established: skipping associated resource deployment reconciliation","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Kibana","namespace":"default","name":"ik-kibana"}
  112. {"log.level":"info","@timestamp":"2020-07-28T08:40:34.029Z","log.logger":"kibana-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":172,"namespace":"default","kibana_name":"ik-kibana","took":0.000691179}
  113. {"log.level":"info","@timestamp":"2020-07-28T08:41:46.692Z","log.logger":"generic-reconciler","message":"Recoverable error during step, continuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","step":"reconcile-cluster-license","error":"failed to revert to basic: Post https://clustername-es-http.default.svc:9200/_license/start_basic?acknowledge=true: dial tcp 172.20.14.193:9200: connect: connection timed out","errorVerbose":"Post https://clustername-es-http.default.svc:9200/_license/start_basic?acknowledge=true: dial tcp 172.20.14.193:9200: connect: connection timed out
  114. failed to revert to basic
  115. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.startBasic
  116.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:78
  117. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.applyLinkedLicense
  118.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:52
  119. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.Reconcile
  120.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/reconcile.go:24
  121. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile.func1
  122.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:203
  123. github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler.(*Results).Apply
  124.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler/results.go:103
  125. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  126.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:196
  127. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  128.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  129. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  130.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  131. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  132.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  133. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  134.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  135. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  136.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  137. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  138.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  139. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  140.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  141. k8s.io/apimachinery/pkg/util/wait.Until
  142.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88
  143. runtime.goexit
  144.     /usr/local/go/src/runtime/asm_amd64.s:1357"}
  145. {"log.level":"error","@timestamp":"2020-07-28T08:41:47.716Z","log.logger":"driver","message":"Could not update remote clusters in Elasticsearch settings","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","error":"Get https://clustername-es-http.default.svc:9200/_cluster/settings: dial tcp 172.20.191.178:9200: connect: connection refused","error.stack_trace":"github.com/go-logr/zapr.(*zapLogger).Error
  146.     /go/pkg/mod/github.com/go-logr/zapr@v0.1.0/zapr.go:128
  147. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  148.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:228
  149. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  150.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  151. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  152.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  153. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  154.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  155. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  156.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  157. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  158.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  159. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  160.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  161. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  162.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  163. k8s.io/apimachinery/pkg/util/wait.Until
  164.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88"}
  165. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.716Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"ConfigMap","namespace":"default","name":"clustername-es-unicast-hosts"}
  166. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.716Z","log.logger":"settings","message":"Creating seed hosts","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","hosts":["10.118.51.195:9300","10.118.54.36:9300","10.118.61.2:9300"]}
  167. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.726Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-secure-settings"}
  168. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.731Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":274,"namespace":"default","es_name":"clustername"}
  169. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.732Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":274,"namespace":"default","es_name":"clustername","took":0.000610797}
  170. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.736Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  171. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.736Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-default-es-config"}
  172. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.742Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":275,"namespace":"default","es_name":"clustername"}
  173. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.742Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":275,"namespace":"default","es_name":"clustername","took":0.000406946}
  174. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.743Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Service","namespace":"default","name":"clustername-es-default"}
  175. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.748Z","log.logger":"generic-reconciler","message":"Updating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"StatefulSet","namespace":"default","name":"clustername-es-default"}
  176. {"log.level":"info","@timestamp":"2020-07-28T08:41:47.758Z","log.logger":"zen2","message":"Ensuring no voting exclusions are set","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  177. {"log.level":"info","@timestamp":"2020-07-28T08:41:48.772Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":467,"namespace":"default","es_name":"clustername","took":132.494073584}
  178. {"log.level":"error","@timestamp":"2020-07-28T08:41:48.772Z","log.logger":"controller-runtime.controller","message":"Reconciler error","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","controller":"elasticsearch-controller","request":"default/clustername","error":"[failed to revert to basic: Post https://clustername-es-http.default.svc:9200/_license/start_basic?acknowledge=true: dial tcp 172.20.14.193:9200: connect: connection timed out, unable to delete /_cluster/voting_config_exclusions: Delete https://clustername-es-http.default.svc:9200/_cluster/voting_config_exclusions?wait_for_removal=false: dial tcp 172.20.191.178:9200: connect: connection refused]","errorCauses":[{"error":"failed to revert to basic: Post https://clustername-es-http.default.svc:9200/_license/start_basic?acknowledge=true: dial tcp 172.20.14.193:9200: connect: connection timed out","errorVerbose":"Post https://clustername-es-http.default.svc:9200/_license/start_basic?acknowledge=true: dial tcp 172.20.14.193:9200: connect: connection timed out
  179. failed to revert to basic
  180. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.startBasic
  181.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:78
  182. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.applyLinkedLicense
  183.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:52
  184. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.Reconcile
  185.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/reconcile.go:24
  186. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile.func1
  187.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:203
  188. github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler.(*Results).Apply
  189.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler/results.go:103
  190. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  191.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:196
  192. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  193.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  194. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  195.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  196. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  197.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  198. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  199.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  200. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  201.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  202. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  203.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  204. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  205.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  206. k8s.io/apimachinery/pkg/util/wait.Until
  207.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88
  208. runtime.goexit
  209.     /usr/local/go/src/runtime/asm_amd64.s:1357"},{"error":"unable to delete /_cluster/voting_config_exclusions: Delete https://clustername-es-http.default.svc:9200/_cluster/voting_config_exclusions?wait_for_removal=false: dial tcp 172.20.191.178:9200: connect: connection refused","errorVerbose":"Delete https://clustername-es-http.default.svc:9200/_cluster/voting_config_exclusions?wait_for_removal=false: dial tcp 172.20.191.178:9200: connect: connection refused
  210. unable to delete /_cluster/voting_config_exclusions
  211. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client.(*clientV7).DeleteVotingConfigExclusions
  212.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client/v7.go:63
  213. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/version/zen2.ClearVotingConfigExclusions
  214.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/version/zen2/voting_exclusions.go:76
  215. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).reconcileNodeSpecs
  216.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/nodes.go:121
  217. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  218.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:263
  219. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  220.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  221. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  222.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  223. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  224.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  225. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  226.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  227. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  228.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  229. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  230.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  231. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  232.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  233. k8s.io/apimachinery/pkg/util/wait.Until
  234.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88
  235. runtime.goexit
  236.     /usr/local/go/src/runtime/asm_amd64.s:1357"}],"error.stack_trace":"github.com/go-logr/zapr.(*zapLogger).Error
  237.     /go/pkg/mod/github.com/go-logr/zapr@v0.1.0/zapr.go:128
  238. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  239.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:258
  240. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  241.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  242. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  243.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  244. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  245.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  246. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  247.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  248. k8s.io/apimachinery/pkg/util/wait.Until
  249.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88"}
  250. {"log.level":"info","@timestamp":"2020-07-28T08:41:48.773Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":470,"namespace":"default","es_name":"clustername"}
  251. {"log.level":"info","@timestamp":"2020-07-28T08:41:48.774Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Service","namespace":"default","name":"clustername-es-transport"}
  252. {"log.level":"info","@timestamp":"2020-07-28T08:41:48.781Z","log.logger":"certificates","message":"No internal CA certificate Secret found, creating a new one","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","owner_namespace":"default","owner_name":"clustername","ca_type":"http"}
  253. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.074Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-http-ca-internal"}
  254. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.091Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":276,"namespace":"default","es_name":"clustername"}
  255. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.091Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":276,"namespace":"default","es_name":"clustername","took":0.000463015}
  256. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.718Z","log.logger":"certificates","message":"Issuing new HTTP certificate","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","secret_name":"clustername-es-http-certs-internal","owner_namespace":"default","owner_name":"clustername"}
  257. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.724Z","log.logger":"certificates","message":"Creating HTTP internal certificate secret","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","secret_name":"clustername-es-http-certs-internal"}
  258. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.731Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-http-certs-public"}
  259. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.732Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":277,"namespace":"default","es_name":"clustername"}
  260. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.732Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":277,"namespace":"default","es_name":"clustername","took":0.000373666}
  261. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.737Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":187,"namespace":"default","kibana_name":"ik-kibana"}
  262. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.752Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":278,"namespace":"default","es_name":"clustername"}
  263. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.752Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":278,"namespace":"default","es_name":"clustername","took":0.000607978}
  264. {"log.level":"info","@timestamp":"2020-07-28T08:41:49.753Z","log.logger":"certificates","message":"No internal CA certificate Secret found, creating a new one","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","owner_namespace":"default","owner_name":"clustername","ca_type":"transport"}
  265. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.135Z","log.logger":"generic-reconciler","message":"Updating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"ik-kibana-kb-es-ca"}
  266. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.172Z","log.logger":"kibana-association-controller","message":"Updating Kibana spec with Elasticsearch backend configuration","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","kibana_name":"ik-kibana"}
  267. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.173Z","log.logger":"kibana-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":173,"namespace":"default","kibana_name":"ik-kibana"}
  268. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.173Z","log.logger":"association","message":"Elasticsearch association not established: skipping associated resource deployment reconciliation","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Kibana","namespace":"default","name":"ik-kibana"}
  269. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.173Z","log.logger":"kibana-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":173,"namespace":"default","kibana_name":"ik-kibana","took":0.000228083}
  270. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.233Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":187,"namespace":"default","kibana_name":"ik-kibana","took":0.496308706}
  271. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.233Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":188,"namespace":"default","kibana_name":"ik-kibana"}
  272. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.266Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-transport-ca-internal"}
  273. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.266Z","log.logger":"kibana-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":174,"namespace":"default","kibana_name":"ik-kibana"}
  274. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.284Z","log.logger":"generic-reconciler","message":"Updating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Deployment","namespace":"default","name":"ik-kibana-kb"}
  275. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.304Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-transport-certs-public"}
  276. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.305Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":279,"namespace":"default","es_name":"clustername"}
  277. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.305Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":279,"namespace":"default","es_name":"clustername","took":0.000298444}
  278. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.344Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-transport-certificates"}
  279. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.345Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":280,"namespace":"default","es_name":"clustername"}
  280. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.345Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":280,"namespace":"default","es_name":"clustername","took":0.000311703}
  281. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.346Z","log.logger":"kibana-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":174,"namespace":"default","kibana_name":"ik-kibana","took":0.079324832}
  282. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.346Z","log.logger":"kibana-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":175,"namespace":"default","kibana_name":"ik-kibana"}
  283. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.367Z","log.logger":"transport","message":"Skipping pod because it has no IP yet","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  284. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.368Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":281,"namespace":"default","es_name":"clustername"}
  285. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.368Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":281,"namespace":"default","es_name":"clustername","took":0.000505997}
  286. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.501Z","log.logger":"kibana-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":175,"namespace":"default","kibana_name":"ik-kibana","took":0.154792658}
  287. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.501Z","log.logger":"kibana-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":176,"namespace":"default","kibana_name":"ik-kibana"}
  288. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.503Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":282,"namespace":"default","es_name":"clustername"}
  289. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.503Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":282,"namespace":"default","es_name":"clustername","took":0.000311774}
  290. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.577Z","log.logger":"kibana-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":176,"namespace":"default","kibana_name":"ik-kibana","took":0.076338285}
  291. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.577Z","log.logger":"kibana-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":177,"namespace":"default","kibana_name":"ik-kibana"}
  292. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.638Z","log.logger":"kibana-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":177,"namespace":"default","kibana_name":"ik-kibana","took":0.060825119}
  293. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.661Z","log.logger":"kibana-association-controller","message":"Updating Kibana spec with Elasticsearch backend configuration","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","kibana_name":"ik-kibana"}
  294. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.698Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":188,"namespace":"default","kibana_name":"ik-kibana","took":0.46498552}
  295. {"log.level":"info","@timestamp":"2020-07-28T08:41:50.698Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":189,"namespace":"default","kibana_name":"ik-kibana"}
  296. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.062Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-internal-users"}
  297. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.085Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":189,"namespace":"default","kibana_name":"ik-kibana","took":0.387098915}
  298. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.085Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":190,"namespace":"default","kibana_name":"ik-kibana"}
  299. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.097Z","log.logger":"generic-reconciler","message":"Creating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"Secret","namespace":"default","name":"clustername-es-xpack-file-realm"}
  300. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.097Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":283,"namespace":"default","es_name":"clustername"}
  301. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.098Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":283,"namespace":"default","es_name":"clustername","took":0.000351325}
  302. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.117Z","log.logger":"observer","message":"Replacing observer HTTP client","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  303. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.117Z","log.logger":"observer","message":"Creating observer for cluster","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  304. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.117Z","log.logger":"generic-reconciler","message":"Updating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"ConfigMap","namespace":"default","name":"clustername-es-unicast-hosts"}
  305. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.137Z","log.logger":"observer","message":"Stopping observer for cluster","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  306. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.137Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":284,"namespace":"default","es_name":"clustername"}
  307. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.138Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":284,"namespace":"default","es_name":"clustername","took":0.000376345}
  308. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.138Z","log.logger":"settings","message":"Seed hosts updated","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","hosts":[]}
  309. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.182Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  310. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.183Z","log.logger":"driver","message":"Performing a forced rolling upgrade","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","statefulset_name":"clustername-es-default","pod_count":1}
  311. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.183Z","log.logger":"driver","message":"Deleting pod for rolling upgrade","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","es_name":"clustername","namespace":"default","pod_name":"clustername-es-default-0","pod_uid":"5385224a-3fb0-4fa1-baed-0f6e035bcbf2"}
  312. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.239Z","log.logger":"license-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":399,"namespace":"default","es_name":"clustername"}
  313. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.258Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":470,"namespace":"default","es_name":"clustername","took":2.485409267}
  314. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.258Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":471,"namespace":"default","es_name":"clustername"}
  315. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.261Z","log.logger":"transport","message":"Skipping pod because it has no IP yet","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  316. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.278Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":285,"namespace":"default","es_name":"clustername"}
  317. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.278Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":285,"namespace":"default","es_name":"clustername","took":0.000346345}
  318. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.279Z","log.logger":"license-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":399,"namespace":"default","es_name":"clustername","took":0.039470749}
  319. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.361Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":190,"namespace":"default","kibana_name":"ik-kibana","took":0.275465243}
  320. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.361Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":191,"namespace":"default","kibana_name":"ik-kibana"}
  321. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.738Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":191,"namespace":"default","kibana_name":"ik-kibana","took":0.377455743}
  322. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.789Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":471,"namespace":"default","es_name":"clustername","took":0.530755701}
  323. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.789Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":472,"namespace":"default","es_name":"clustername"}
  324. {"log.level":"info","@timestamp":"2020-07-28T08:41:51.791Z","log.logger":"transport","message":"Skipping pod because it has no IP yet","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  325. {"log.level":"info","@timestamp":"2020-07-28T08:41:52.092Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":472,"namespace":"default","es_name":"clustername","took":0.302637037}
  326. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.232Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":473,"namespace":"default","es_name":"clustername"}
  327. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.236Z","log.logger":"transport","message":"Skipping pod because it has no IP yet","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  328. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.570Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  329. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.571Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  330. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.571Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":473,"namespace":"default","es_name":"clustername","took":0.339454662}
  331. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.571Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":474,"namespace":"default","es_name":"clustername"}
  332. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.575Z","log.logger":"transport","message":"Skipping pod because it has no IP yet","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  333. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.877Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  334. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.878Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  335. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.878Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":474,"namespace":"default","es_name":"clustername","took":0.306931921}
  336. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.886Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":475,"namespace":"default","es_name":"clustername"}
  337. {"log.level":"info","@timestamp":"2020-07-28T08:41:56.889Z","log.logger":"transport","message":"Skipping pod because it has no IP yet","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  338. {"log.level":"info","@timestamp":"2020-07-28T08:41:57.203Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  339. {"log.level":"info","@timestamp":"2020-07-28T08:41:57.204Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  340. {"log.level":"info","@timestamp":"2020-07-28T08:41:57.204Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":475,"namespace":"default","es_name":"clustername","took":0.318524377}
  341. {"log.level":"info","@timestamp":"2020-07-28T08:42:01.258Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":476,"namespace":"default","es_name":"clustername"}
  342. {"log.level":"info","@timestamp":"2020-07-28T08:42:01.261Z","log.logger":"transport","message":"Skipping pod because it has no IP yet","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  343. {"log.level":"info","@timestamp":"2020-07-28T08:42:01.561Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  344. {"log.level":"info","@timestamp":"2020-07-28T08:42:01.562Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  345. {"log.level":"info","@timestamp":"2020-07-28T08:42:01.562Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":476,"namespace":"default","es_name":"clustername","took":0.304056315}
  346. {"log.level":"info","@timestamp":"2020-07-28T08:42:11.563Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":477,"namespace":"default","es_name":"clustername"}
  347. {"log.level":"info","@timestamp":"2020-07-28T08:42:11.567Z","log.logger":"transport","message":"Skipping pod because it has no IP yet","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  348. {"log.level":"info","@timestamp":"2020-07-28T08:42:11.858Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  349. {"log.level":"info","@timestamp":"2020-07-28T08:42:11.859Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  350. {"log.level":"info","@timestamp":"2020-07-28T08:42:11.859Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":477,"namespace":"default","es_name":"clustername","took":0.296329758}
  351. {"log.level":"info","@timestamp":"2020-07-28T08:42:14.561Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":478,"namespace":"default","es_name":"clustername"}
  352. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.344Z","log.logger":"transport","message":"No tls certificate found in secret","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","pod_name":"clustername-es-default-0"}
  353. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.344Z","log.logger":"transport","message":"Issuing new certificate","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","pod_name":"clustername-es-default-0"}
  354. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.358Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":286,"namespace":"default","es_name":"clustername"}
  355. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.359Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":286,"namespace":"default","es_name":"clustername","took":0.000484047}
  356. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.668Z","log.logger":"generic-reconciler","message":"Updating resource","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","kind":"ConfigMap","namespace":"default","name":"clustername-es-unicast-hosts"}
  357. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.674Z","log.logger":"settings","message":"Seed hosts updated","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","hosts":["10.118.48.74:9300"]}
  358. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.689Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  359. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.690Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  360. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.690Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":478,"namespace":"default","es_name":"clustername","took":1.129360751}
  361. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.690Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":479,"namespace":"default","es_name":"clustername"}
  362. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.989Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  363. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.990Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  364. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.990Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":479,"namespace":"default","es_name":"clustername","took":0.300014744}
  365. {"log.level":"info","@timestamp":"2020-07-28T08:42:15.990Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":480,"namespace":"default","es_name":"clustername"}
  366. {"log.level":"info","@timestamp":"2020-07-28T08:42:16.281Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  367. {"log.level":"info","@timestamp":"2020-07-28T08:42:16.282Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  368. {"log.level":"info","@timestamp":"2020-07-28T08:42:16.282Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":480,"namespace":"default","es_name":"clustername","took":0.291843621}
  369. {"log.level":"info","@timestamp":"2020-07-28T08:42:16.706Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":481,"namespace":"default","es_name":"clustername"}
  370. {"log.level":"info","@timestamp":"2020-07-28T08:42:17.042Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  371. {"log.level":"info","@timestamp":"2020-07-28T08:42:17.043Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  372. {"log.level":"info","@timestamp":"2020-07-28T08:42:17.044Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":481,"namespace":"default","es_name":"clustername","took":0.33740151}
  373. {"log.level":"info","@timestamp":"2020-07-28T08:42:17.835Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":482,"namespace":"default","es_name":"clustername"}
  374. {"log.level":"info","@timestamp":"2020-07-28T08:42:18.133Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  375. {"log.level":"info","@timestamp":"2020-07-28T08:42:18.134Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  376. {"log.level":"info","@timestamp":"2020-07-28T08:42:18.134Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":482,"namespace":"default","es_name":"clustername","took":0.298775869}
  377. {"log.level":"info","@timestamp":"2020-07-28T08:42:21.859Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":483,"namespace":"default","es_name":"clustername"}
  378. {"log.level":"info","@timestamp":"2020-07-28T08:42:22.173Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  379. {"log.level":"info","@timestamp":"2020-07-28T08:42:22.173Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  380. {"log.level":"info","@timestamp":"2020-07-28T08:42:22.173Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":483,"namespace":"default","es_name":"clustername","took":0.314276911}
  381. {"log.level":"info","@timestamp":"2020-07-28T08:42:22.174Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":484,"namespace":"default","es_name":"clustername"}
  382. {"log.level":"info","@timestamp":"2020-07-28T08:42:22.473Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  383. {"log.level":"info","@timestamp":"2020-07-28T08:42:22.474Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  384. {"log.level":"info","@timestamp":"2020-07-28T08:42:22.474Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":484,"namespace":"default","es_name":"clustername","took":0.300840344}
  385. {"log.level":"info","@timestamp":"2020-07-28T08:42:23.220Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":485,"namespace":"default","es_name":"clustername"}
  386. {"log.level":"info","@timestamp":"2020-07-28T08:42:23.523Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  387. {"log.level":"info","@timestamp":"2020-07-28T08:42:23.524Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  388. {"log.level":"info","@timestamp":"2020-07-28T08:42:23.524Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":485,"namespace":"default","es_name":"clustername","took":0.304024154}
  389. {"log.level":"info","@timestamp":"2020-07-28T08:42:24.322Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":486,"namespace":"default","es_name":"clustername"}
  390. {"log.level":"info","@timestamp":"2020-07-28T08:42:24.625Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  391. {"log.level":"info","@timestamp":"2020-07-28T08:42:24.626Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  392. {"log.level":"info","@timestamp":"2020-07-28T08:42:24.627Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":486,"namespace":"default","es_name":"clustername","took":0.304642534}
  393. {"log.level":"info","@timestamp":"2020-07-28T08:42:30.697Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":487,"namespace":"default","es_name":"clustername"}
  394. {"log.level":"info","@timestamp":"2020-07-28T08:42:31.004Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  395. {"log.level":"info","@timestamp":"2020-07-28T08:42:31.005Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  396. {"log.level":"info","@timestamp":"2020-07-28T08:42:31.005Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":487,"namespace":"default","es_name":"clustername","took":0.308582243}
  397. {"log.level":"info","@timestamp":"2020-07-28T08:42:31.848Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":488,"namespace":"default","es_name":"clustername"}
  398. {"log.level":"info","@timestamp":"2020-07-28T08:42:32.163Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  399. {"log.level":"info","@timestamp":"2020-07-28T08:42:32.164Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  400. {"log.level":"info","@timestamp":"2020-07-28T08:42:32.164Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":488,"namespace":"default","es_name":"clustername","took":0.315651713}
  401. {"log.level":"info","@timestamp":"2020-07-28T08:42:32.174Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":489,"namespace":"default","es_name":"clustername"}
  402. {"log.level":"info","@timestamp":"2020-07-28T08:42:32.482Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  403. {"log.level":"info","@timestamp":"2020-07-28T08:42:32.483Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  404. {"log.level":"info","@timestamp":"2020-07-28T08:42:32.483Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":489,"namespace":"default","es_name":"clustername","took":0.309312622}
  405. {"log.level":"info","@timestamp":"2020-07-28T08:42:42.483Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":490,"namespace":"default","es_name":"clustername"}
  406. {"log.level":"info","@timestamp":"2020-07-28T08:42:42.820Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  407. {"log.level":"info","@timestamp":"2020-07-28T08:42:42.821Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  408. {"log.level":"info","@timestamp":"2020-07-28T08:42:42.821Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":490,"namespace":"default","es_name":"clustername","took":0.338071964}
  409. {"log.level":"info","@timestamp":"2020-07-28T08:42:52.822Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":491,"namespace":"default","es_name":"clustername"}
  410. {"log.level":"info","@timestamp":"2020-07-28T08:42:53.129Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  411. {"log.level":"info","@timestamp":"2020-07-28T08:42:53.130Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  412. {"log.level":"info","@timestamp":"2020-07-28T08:42:53.130Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":491,"namespace":"default","es_name":"clustername","took":0.308631537}
  413. {"log.level":"info","@timestamp":"2020-07-28T08:43:03.130Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":492,"namespace":"default","es_name":"clustername"}
  414. {"log.level":"info","@timestamp":"2020-07-28T08:43:03.434Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  415. {"log.level":"info","@timestamp":"2020-07-28T08:43:03.435Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  416. {"log.level":"info","@timestamp":"2020-07-28T08:43:03.435Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":492,"namespace":"default","es_name":"clustername","took":0.304960561}
  417. {"log.level":"info","@timestamp":"2020-07-28T08:43:13.436Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":493,"namespace":"default","es_name":"clustername"}
  418. {"log.level":"info","@timestamp":"2020-07-28T08:43:13.751Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  419. {"log.level":"info","@timestamp":"2020-07-28T08:43:13.752Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  420. {"log.level":"info","@timestamp":"2020-07-28T08:43:13.752Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":493,"namespace":"default","es_name":"clustername","took":0.316062094}
  421. {"log.level":"info","@timestamp":"2020-07-28T08:43:23.706Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":494,"namespace":"default","es_name":"clustername"}
  422. {"log.level":"info","@timestamp":"2020-07-28T08:43:55.611Z","log.logger":"generic-reconciler","message":"Recoverable error during step, continuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","step":"reconcile-cluster-license","error":"failed to revert to basic: 503 Service Unavailable: unknown","errorVerbose":"503 Service Unavailable: unknown
  423. failed to revert to basic
  424. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.startBasic
  425.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:78
  426. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.applyLinkedLicense
  427.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:52
  428. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.Reconcile
  429.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/reconcile.go:24
  430. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile.func1
  431.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:203
  432. github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler.(*Results).Apply
  433.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler/results.go:103
  434. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  435.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:196
  436. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  437.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  438. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  439.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  440. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  441.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  442. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  443.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  444. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  445.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  446. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  447.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  448. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  449.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  450. k8s.io/apimachinery/pkg/util/wait.Until
  451.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88
  452. runtime.goexit
  453.     /usr/local/go/src/runtime/asm_amd64.s:1357"}
  454. {"log.level":"error","@timestamp":"2020-07-28T08:43:56.644Z","log.logger":"driver","message":"Could not update remote clusters in Elasticsearch settings","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername","error":"Get https://clustername-es-http.default.svc:9200/_cluster/settings: dial tcp 172.20.191.178:9200: connect: connection refused","error.stack_trace":"github.com/go-logr/zapr.(*zapLogger).Error
  455.     /go/pkg/mod/github.com/go-logr/zapr@v0.1.0/zapr.go:128
  456. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  457.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:228
  458. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  459.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  460. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  461.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  462. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  463.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  464. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  465.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  466. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  467.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  468. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  469.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  470. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  471.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  472. k8s.io/apimachinery/pkg/util/wait.Until
  473.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88"}
  474. {"log.level":"info","@timestamp":"2020-07-28T08:43:56.649Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  475. {"log.level":"info","@timestamp":"2020-07-28T08:43:56.650Z","log.logger":"zen2","message":"Ensuring no voting exclusions are set","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  476. {"log.level":"info","@timestamp":"2020-07-28T08:43:57.676Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":494,"namespace":"default","es_name":"clustername","took":33.969446522}
  477. {"log.level":"error","@timestamp":"2020-07-28T08:43:57.676Z","log.logger":"controller-runtime.controller","message":"Reconciler error","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","controller":"elasticsearch-controller","request":"default/clustername","error":"[failed to revert to basic: 503 Service Unavailable: unknown, unable to delete /_cluster/voting_config_exclusions: Delete https://clustername-es-http.default.svc:9200/_cluster/voting_config_exclusions?wait_for_removal=false: dial tcp 172.20.191.178:9200: connect: connection refused]","errorCauses":[{"error":"failed to revert to basic: 503 Service Unavailable: unknown","errorVerbose":"503 Service Unavailable: unknown
  478. failed to revert to basic
  479. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.startBasic
  480.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:78
  481. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.applyLinkedLicense
  482.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:52
  483. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.Reconcile
  484.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/reconcile.go:24
  485. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile.func1
  486.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:203
  487. github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler.(*Results).Apply
  488.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler/results.go:103
  489. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  490.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:196
  491. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  492.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  493. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  494.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  495. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  496.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  497. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  498.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  499. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  500.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  501. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  502.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  503. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  504.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  505. k8s.io/apimachinery/pkg/util/wait.Until
  506.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88
  507. runtime.goexit
  508.     /usr/local/go/src/runtime/asm_amd64.s:1357"},{"error":"unable to delete /_cluster/voting_config_exclusions: Delete https://clustername-es-http.default.svc:9200/_cluster/voting_config_exclusions?wait_for_removal=false: dial tcp 172.20.191.178:9200: connect: connection refused","errorVerbose":"Delete https://clustername-es-http.default.svc:9200/_cluster/voting_config_exclusions?wait_for_removal=false: dial tcp 172.20.191.178:9200: connect: connection refused
  509. unable to delete /_cluster/voting_config_exclusions
  510. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client.(*clientV7).DeleteVotingConfigExclusions
  511.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/client/v7.go:63
  512. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/version/zen2.ClearVotingConfigExclusions
  513.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/version/zen2/voting_exclusions.go:76
  514. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).reconcileNodeSpecs
  515.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/nodes.go:121
  516. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  517.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:263
  518. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  519.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  520. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  521.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  522. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  523.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  524. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  525.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  526. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  527.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  528. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  529.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  530. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  531.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  532. k8s.io/apimachinery/pkg/util/wait.Until
  533.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88
  534. runtime.goexit
  535.     /usr/local/go/src/runtime/asm_amd64.s:1357"}],"error.stack_trace":"github.com/go-logr/zapr.(*zapLogger).Error
  536.     /go/pkg/mod/github.com/go-logr/zapr@v0.1.0/zapr.go:128
  537. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  538.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:258
  539. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  540.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  541. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  542.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  543. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  544.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  545. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  546.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  547. k8s.io/apimachinery/pkg/util/wait.Until
  548.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88"}
  549. {"log.level":"info","@timestamp":"2020-07-28T08:43:57.676Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":495,"namespace":"default","es_name":"clustername"}
  550. {"log.level":"info","@timestamp":"2020-07-28T08:43:57.701Z","log.logger":"license-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":400,"namespace":"default","es_name":"clustername"}
  551. {"log.level":"info","@timestamp":"2020-07-28T08:43:57.702Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":192,"namespace":"default","kibana_name":"ik-kibana"}
  552. {"log.level":"info","@timestamp":"2020-07-28T08:43:57.721Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":287,"namespace":"default","es_name":"clustername"}
  553. {"log.level":"info","@timestamp":"2020-07-28T08:43:57.722Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":287,"namespace":"default","es_name":"clustername","took":0.000470686}
  554. {"log.level":"info","@timestamp":"2020-07-28T08:43:57.742Z","log.logger":"license-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":400,"namespace":"default","es_name":"clustername","took":0.04033845}
  555. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.083Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":192,"namespace":"default","kibana_name":"ik-kibana","took":0.381470308}
  556. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.175Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  557. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.176Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  558. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.176Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":495,"namespace":"default","es_name":"clustername","took":0.5003569}
  559. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.176Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":496,"namespace":"default","es_name":"clustername"}
  560. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.488Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  561. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.489Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  562. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.496Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":496,"namespace":"default","es_name":"clustername","took":0.319602176}
  563. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.497Z","log.logger":"license-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":401,"namespace":"default","es_name":"clustername"}
  564. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.497Z","log.logger":"kibana-association-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":193,"namespace":"default","kibana_name":"ik-kibana"}
  565. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.512Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":497,"namespace":"default","es_name":"clustername"}
  566. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.532Z","log.logger":"remotecluster-remoteca","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":288,"namespace":"default","es_name":"clustername"}
  567. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.533Z","log.logger":"remotecluster-remoteca","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":288,"namespace":"default","es_name":"clustername","took":0.000484746}
  568. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.533Z","log.logger":"license-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":401,"namespace":"default","es_name":"clustername","took":0.036295498}
  569. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.897Z","log.logger":"kibana-association-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":193,"namespace":"default","kibana_name":"ik-kibana","took":0.399635278}
  570. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.994Z","log.logger":"driver","message":"Limiting master nodes creation to one at a time","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","statefulset_name":"clustername-es-default","target":3,"actual":1}
  571. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.995Z","log.logger":"driver","message":"ES cannot be reached yet, re-queuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","namespace":"default","es_name":"clustername"}
  572. {"log.level":"info","@timestamp":"2020-07-28T08:43:58.995Z","log.logger":"elasticsearch-controller","message":"Ending reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":497,"namespace":"default","es_name":"clustername","took":0.483509196}
  573. {"log.level":"info","@timestamp":"2020-07-28T08:44:00.655Z","log.logger":"elasticsearch-controller","message":"Starting reconciliation run","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","iteration":498,"namespace":"default","es_name":"clustername"}
  574. {"log.level":"info","@timestamp":"2020-07-28T08:44:32.984Z","log.logger":"generic-reconciler","message":"Recoverable error during step, continuing","service.version":"1.1.1-f13b6d26","service.type":"eck","ecs.version":"1.4.0","step":"reconcile-cluster-license","error":"failed to revert to basic: 503 Service Unavailable: unknown","errorVerbose":"503 Service Unavailable: unknown
  575. failed to revert to basic
  576. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.startBasic
  577.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:78
  578. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.applyLinkedLicense
  579.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/apply.go:52
  580. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license.Reconcile
  581.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/license/reconcile.go:24
  582. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile.func1
  583.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:203
  584. github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler.(*Results).Apply
  585.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler/results.go:103
  586. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver.(*defaultDriver).Reconcile
  587.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/driver/driver.go:196
  588. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).internalReconcile
  589.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:308
  590. github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch.(*ReconcileElasticsearch).Reconcile
  591.     /go/src/github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/elasticsearch_controller.go:215
  592. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
  593.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256
  594. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
  595.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232
  596. sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker
  597.     /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211
  598. k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1
  599.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:152
  600. k8s.io/apimachinery/pkg/util/wait.JitterUntil
  601.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153
  602. k8s.io/apimachinery/pkg/util/wait.Until
  603.     /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88
  604. runtime.goexit
  605.     /usr/local/go/src/runtime/asm_amd64.s:1357"}
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!