API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Oct 16th, 2019
171
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 71.84 KB | None | 0 0
raw download clone embed print report
  1. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt",
  2. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  3. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  4. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  5. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  6. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  7. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  8. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  9. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  10. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  11. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  12. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  13. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  14. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  15. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  16. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  17. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  18. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  19. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  20. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  21. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  22. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  23. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  24. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  25. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  26. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  27. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  28. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  29. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  30. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  31. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  32. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  33. "... 4 more"] }
  34. {"type": "server", "timestamp": "2019-10-11T20:32:25,636+0000", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "[[logstash-2019.10.03][0]] marking and sending shard failed due to [failed recovery]" ,
  35. "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}",
  36. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) [elasticsearch-7.0.0.jar:7.0.0]",
  37. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0]",
  38. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  39. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  40. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  41. "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway",
  42. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]",
  43. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  44. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  45. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  46. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  47. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  48. "... 4 more",
  49. "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine",
  50. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]",
  51. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  52. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  53. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  54. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  55. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  56. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  57. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  58. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  59. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  60. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  61. "... 4 more",
  62. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs",
  63. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  64. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  65. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  66. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  67. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  68. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  69. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  70. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  71. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  72. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  73. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  74. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  75. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  76. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  77. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  78. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  79. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  80. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  81. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  82. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  83. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  84. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  85. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  86. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  87. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  88. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  89. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  90. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  91. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  92. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  93. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  94. "... 4 more"] }
  95. {"type": "server", "timestamp": "2019-10-11T20:32:25,639+0000", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "failing shard [failed shard, shard [logstash-2019.10.03][0], node[gjegRSM1Rbi22HYJOFYINw], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=U2qBx23_S6268iPdPqnwhA], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-10-11T20:32:00.066Z], failed_attempts[3], delayed=false, details[failed shard on node [gjegRSM1Rbi22HYJOFYINw]: failed recovery, failure RecoveryFailedException[[logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs]; ], allocation_status[fetching_shard_data]], message [failed recovery], failure [RecoveryFailedException[[logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs]; ], markAsStale [true]]" ,
  96. "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}",
  97. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) ~[elasticsearch-7.0.0.jar:7.0.0]",
  98. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  99. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  100. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  101. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  102. "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway",
  103. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]",
  104. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  105. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  106. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  107. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  108. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  109. "... 4 more",
  110. "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine",
  111. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]",
  112. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  113. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  114. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  115. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  116. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  117. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  118. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  119. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  120. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  121. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  122. "... 4 more",
  123. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs",
  124. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  125. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  126. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  127. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  128. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  129. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  130. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  131. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  132. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  133. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  134. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  135. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  136. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  137. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  138. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  139. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  140. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  141. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  142. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  143. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  144. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  145. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  146. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  147. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  148. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  149. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  150. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  151. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  152. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  153. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  154. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  155. "... 4 more"] }
  156. {"type": "server", "timestamp": "2019-10-11T20:32:29,708+0000", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "[[logstash-2019.10.04][0]] marking and sending shard failed due to [failed recovery]" ,
  157. "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}",
  158. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) [elasticsearch-7.0.0.jar:7.0.0]",
  159. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0]",
  160. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  161. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  162. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  163. "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway",
  164. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]",
  165. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  166. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  167. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  168. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  169. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  170. "... 4 more",
  171. "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine",
  172. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]",
  173. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  174. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  175. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  176. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  177. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  178. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  179. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  180. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  181. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  182. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  183. "... 4 more",
  184. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt",
  185. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  186. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  187. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  188. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  189. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  190. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  191. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  192. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  193. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  194. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  195. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  196. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  197. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  198. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  199. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  200. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  201. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  202. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  203. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  204. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  205. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  206. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  207. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  208. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  209. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  210. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  211. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  212. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  213. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  214. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  215. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  216. "... 4 more"] }
  217. {"type": "server", "timestamp": "2019-10-11T20:32:30,032+0000", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "failing shard [failed shard, shard [logstash-2019.10.04][0], node[gjegRSM1Rbi22HYJOFYINw], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=Xs0xcNPyQdORV_hS5JUkHg], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-10-11T20:32:02.425Z], failed_attempts[3], delayed=false, details[failed shard on node [gjegRSM1Rbi22HYJOFYINw]: failed recovery, failure RecoveryFailedException[[logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt]; ], allocation_status[no_valid_shard_copy]], message [failed recovery], failure [RecoveryFailedException[[logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt]; ], markAsStale [true]]" ,
  218. "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}",
  219. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) ~[elasticsearch-7.0.0.jar:7.0.0]",
  220. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  221. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  222. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  223. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  224. "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway",
  225. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]",
  226. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  227. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  228. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  229. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  230. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  231. "... 4 more",
  232. "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine",
  233. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]",
  234. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  235. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  236. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  237. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  238. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  239. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  240. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  241. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  242. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  243. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  244. "... 4 more",
  245. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt",
  246. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  247. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  248. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  249. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  250. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  251. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  252. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  253. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  254. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  255. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  256. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  257. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  258. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  259. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  260. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  261. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  262. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  263. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  264. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  265. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  266. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  267. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  268. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  269. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  270. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  271. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  272. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  273. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  274. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  275. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  276. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  277. "... 4 more"] }
  278. {"type": "server", "timestamp": "2019-10-11T20:32:52,397+0000", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "[[logstash-2019.10.03][0]] marking and sending shard failed due to [failed recovery]" ,
  279. "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}",
  280. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) [elasticsearch-7.0.0.jar:7.0.0]",
  281. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0]",
  282. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  283. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  284. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  285. "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway",
  286. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]",
  287. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  288. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  289. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  290. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  291. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  292. "... 4 more",
  293. "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine",
  294. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]",
  295. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  296. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  297. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  298. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  299. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  300. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  301. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  302. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  303. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  304. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  305. "... 4 more",
  306. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs",
  307. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  308. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  309. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  310. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  311. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  312. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  313. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  314. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  315. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  316. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  317. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  318. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  319. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  320. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  321. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  322. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  323. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  324. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  325. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  326. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  327. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  328. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  329. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  330. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  331. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  332. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  333. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  334. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  335. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  336. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  337. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  338. "... 4 more"] }
  339. {"type": "server", "timestamp": "2019-10-11T20:32:52,400+0000", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "failing shard [failed shard, shard [logstash-2019.10.03][0], node[gjegRSM1Rbi22HYJOFYINw], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=U2qBx23_S6268iPdPqnwhA], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-10-11T20:32:25.638Z], failed_attempts[4], delayed=false, details[failed shard on node [gjegRSM1Rbi22HYJOFYINw]: failed recovery, failure RecoveryFailedException[[logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs]; ], allocation_status[fetching_shard_data]], message [failed recovery], failure [RecoveryFailedException[[logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs]; ], markAsStale [true]]" ,
  340. "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}",
  341. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) ~[elasticsearch-7.0.0.jar:7.0.0]",
  342. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  343. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  344. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  345. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  346. "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway",
  347. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]",
  348. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  349. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  350. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  351. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  352. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  353. "... 4 more",
  354. "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine",
  355. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]",
  356. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  357. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  358. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  359. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  360. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  361. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  362. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  363. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  364. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  365. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  366. "... 4 more",
  367. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs",
  368. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  369. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  370. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  371. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  372. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  373. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  374. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  375. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  376. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  377. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  378. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  379. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  380. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  381. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  382. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  383. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  384. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  385. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  386. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  387. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  388. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  389. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  390. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  391. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  392. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  393. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  394. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  395. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  396. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  397. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  398. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  399. "... 4 more"] }
  400. {"type": "server", "timestamp": "2019-10-11T20:32:56,928+0000", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "[[logstash-2019.10.04][0]] marking and sending shard failed due to [failed recovery]" ,
  401. "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}",
  402. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) [elasticsearch-7.0.0.jar:7.0.0]",
  403. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0]",
  404. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  405. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  406. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  407. "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway",
  408. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]",
  409. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  410. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  411. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  412. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  413. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  414. "... 4 more",
  415. "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine",
  416. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]",
  417. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  418. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  419. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  420. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  421. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  422. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  423. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  424. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  425. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  426. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  427. "... 4 more",
  428. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt",
  429. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  430. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  431. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  432. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  433. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  434. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  435. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  436. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  437. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  438. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  439. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  440. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  441. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  442. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  443. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  444. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  445. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  446. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  447. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  448. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  449. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  450. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  451. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  452. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  453. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  454. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  455. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  456. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  457. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  458. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  459. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  460. "... 4 more"] }
  461. {"type": "server", "timestamp": "2019-10-11T20:32:57,242+0000", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "failing shard [failed shard, shard [logstash-2019.10.04][0], node[gjegRSM1Rbi22HYJOFYINw], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=Xs0xcNPyQdORV_hS5JUkHg], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-10-11T20:32:30.032Z], failed_attempts[4], delayed=false, details[failed shard on node [gjegRSM1Rbi22HYJOFYINw]: failed recovery, failure RecoveryFailedException[[logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt]; ], allocation_status[no_valid_shard_copy]], message [failed recovery], failure [RecoveryFailedException[[logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt]; ], markAsStale [true]]" ,
  462. "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}",
  463. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) ~[elasticsearch-7.0.0.jar:7.0.0]",
  464. "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]",
  465. "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]",
  466. "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]",
  467. "at java.lang.Thread.run(Thread.java:835) [?:?]",
  468. "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway",
  469. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]",
  470. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  471. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  472. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  473. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  474. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  475. "... 4 more",
  476. "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine",
  477. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]",
  478. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  479. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  480. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  481. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  482. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  483. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  484. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  485. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  486. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  487. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  488. "... 4 more",
  489. "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt",
  490. "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]",
  491. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]",
  492. "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]",
  493. "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]",
  494. "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]",
  495. "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]",
  496. "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  497. "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  498. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  499. "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]",
  500. "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  501. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]",
  502. "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]",
  503. "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  504. "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  505. "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  506. "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  507. "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]",
  508. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]",
  509. "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]",
  510. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]",
  511. "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]",
  512. "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]",
  513. "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]",
  514. "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]",
  515. "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]",
  516. "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]",
  517. "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]",
  518. "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]",
  519. "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]",
  520. "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]",
  521. "... 4 more"] }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!