Pastebin
API
tools
faq
paste
Login
Sign up
Please fix the following errors:
New Paste
Syntax Highlighting
"Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] } {"type": "server", "timestamp": "2019-10-11T20:32:25,636+0000", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "[[logstash-2019.10.03][0]] marking and sending shard failed due to [failed recovery]" , "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) [elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0]", "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]", "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]", "at java.lang.Thread.run(Thread.java:835) [?:?]", "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] } {"type": "server", "timestamp": "2019-10-11T20:32:25,639+0000", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "failing shard [failed shard, shard [logstash-2019.10.03][0], node[gjegRSM1Rbi22HYJOFYINw], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=U2qBx23_S6268iPdPqnwhA], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-10-11T20:32:00.066Z], failed_attempts[3], delayed=false, details[failed shard on node [gjegRSM1Rbi22HYJOFYINw]: failed recovery, failure RecoveryFailedException[[logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs]; ], allocation_status[fetching_shard_data]], message [failed recovery], failure [RecoveryFailedException[[logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs]; ], markAsStale [true]]" , "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]", "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]", "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]", "at java.lang.Thread.run(Thread.java:835) [?:?]", "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] } {"type": "server", "timestamp": "2019-10-11T20:32:29,708+0000", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "[[logstash-2019.10.04][0]] marking and sending shard failed due to [failed recovery]" , "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) [elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0]", "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]", "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]", "at java.lang.Thread.run(Thread.java:835) [?:?]", "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] } {"type": "server", "timestamp": "2019-10-11T20:32:30,032+0000", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "failing shard [failed shard, shard [logstash-2019.10.04][0], node[gjegRSM1Rbi22HYJOFYINw], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=Xs0xcNPyQdORV_hS5JUkHg], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-10-11T20:32:02.425Z], failed_attempts[3], delayed=false, details[failed shard on node [gjegRSM1Rbi22HYJOFYINw]: failed recovery, failure RecoveryFailedException[[logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt]; ], allocation_status[no_valid_shard_copy]], message [failed recovery], failure [RecoveryFailedException[[logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt]; ], markAsStale [true]]" , "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]", "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]", "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]", "at java.lang.Thread.run(Thread.java:835) [?:?]", "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] } {"type": "server", "timestamp": "2019-10-11T20:32:52,397+0000", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "[[logstash-2019.10.03][0]] marking and sending shard failed due to [failed recovery]" , "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) [elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0]", "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]", "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]", "at java.lang.Thread.run(Thread.java:835) [?:?]", "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] } {"type": "server", "timestamp": "2019-10-11T20:32:52,400+0000", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "failing shard [failed shard, shard [logstash-2019.10.03][0], node[gjegRSM1Rbi22HYJOFYINw], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=U2qBx23_S6268iPdPqnwhA], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-10-11T20:32:25.638Z], failed_attempts[4], delayed=false, details[failed shard on node [gjegRSM1Rbi22HYJOFYINw]: failed recovery, failure RecoveryFailedException[[logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs]; ], allocation_status[fetching_shard_data]], message [failed recovery], failure [RecoveryFailedException[[logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs]; ], markAsStale [true]]" , "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.03][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]", "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]", "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]", "at java.lang.Thread.run(Thread.java:835) [?:?]", "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/Qcz9ztHhQaCjSvUFuF3VlQ/0/index/_311.cfs", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] } {"type": "server", "timestamp": "2019-10-11T20:32:56,928+0000", "level": "WARN", "component": "o.e.i.c.IndicesClusterStateService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "[[logstash-2019.10.04][0]] marking and sending shard failed due to [failed recovery]" , "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) [elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) [elasticsearch-7.0.0.jar:7.0.0]", "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]", "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]", "at java.lang.Thread.run(Thread.java:835) [?:?]", "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] } {"type": "server", "timestamp": "2019-10-11T20:32:57,242+0000", "level": "WARN", "component": "o.e.c.r.a.AllocationService", "cluster.name": "docker-cluster", "node.name": "3730edbdef97", "cluster.uuid": "69cIMsbUTUOiK3MOdNggow", "node.id": "gjegRSM1Rbi22HYJOFYINw", "message": "failing shard [failed shard, shard [logstash-2019.10.04][0], node[gjegRSM1Rbi22HYJOFYINw], [P], recovery_source[existing store recovery; bootstrap_history_uuid=false], s[INITIALIZING], a[id=Xs0xcNPyQdORV_hS5JUkHg], unassigned_info[[reason=ALLOCATION_FAILED], at[2019-10-11T20:32:30.032Z], failed_attempts[4], delayed=false, details[failed shard on node [gjegRSM1Rbi22HYJOFYINw]: failed recovery, failure RecoveryFailedException[[logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt]; ], allocation_status[no_valid_shard_copy]], message [failed recovery], failure [RecoveryFailedException[[logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}]; nested: IndexShardRecoveryException[failed to recover from gateway]; nested: EngineCreationFailureException[failed to create engine]; nested: NoSuchFileException[/usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt]; ], markAsStale [true]]" , "stacktrace": ["org.elasticsearch.indices.recovery.RecoveryFailedException: [logstash-2019.10.04][0]: Recovery failed on {3730edbdef97}{gjegRSM1Rbi22HYJOFYINw}{QP5gwwEgTKm44DzJisNfaQ}{172.17.0.2}{172.17.0.2:9300}{ml.machine_memory=4294967296, xpack.installed=true, ml.max_open_jobs=20}", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2343) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:681) ~[elasticsearch-7.0.0.jar:7.0.0]", "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]", "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]", "at java.lang.Thread.run(Thread.java:835) [?:?]", "Caused by: org.elasticsearch.index.shard.IndexShardRecoveryException: failed to recover from gateway", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:438) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: org.elasticsearch.index.engine.EngineCreationFailureException: failed to create engine", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:204) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more", "Caused by: java.nio.file.NoSuchFileException: /usr/share/elasticsearch/data/nodes/0/indices/np3g96ylRuGrsJKZ4Zo2LA/0/index/_17i.fdt", "at sun.nio.fs.UnixException.translateToIOException(UnixException.java:92) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:111) ~[?:?]", "at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:116) ~[?:?]", "at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:249) ~[?:?]", "at sun.nio.fs.AbstractFileSystemProvider.delete(AbstractFileSystemProvider.java:105) ~[?:?]", "at java.nio.file.Files.delete(Files.java:1144) ~[?:?]", "at org.apache.lucene.store.FSDirectory.privateDeleteFile(FSDirectory.java:371) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FSDirectory.deleteFile(FSDirectory.java:340) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.ByteSizeCachingDirectory.deleteFile(ByteSizeCachingDirectory.java:176) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.FilterDirectory.deleteFile(FilterDirectory.java:63) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:735) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.store.Store$StoreDirectory.deleteFile(Store.java:740) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.apache.lucene.store.LockValidatingDirectoryWrapper.deleteFile(LockValidatingDirectoryWrapper.java:38) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:696) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.deleteFiles(IndexFileDeleter.java:690) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexFileDeleter.<init>(IndexFileDeleter.java:238) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:898) ~[lucene-core-8.0.0.jar:8.0.0 2ae4746365c1ee72a0047ced7610b2096e438979 - jimczi - 2019-03-08 11:58:55]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2127) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.createWriter(InternalEngine.java:2115) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:199) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:165) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1447) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1397) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:433) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:95) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.executeRecovery(StoreRecovery.java:308) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:93) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1686) ~[elasticsearch-7.0.0.jar:7.0.0]", "at org.elasticsearch.index.shard.IndexShard.lambda$startRecovery$9(IndexShard.java:2339) ~[elasticsearch-7.0.0.jar:7.0.0]", "... 4 more"] }
Optional Paste Settings
Category:
None
Cryptocurrency
Cybersecurity
Fixit
Food
Gaming
Haiku
Help
History
Housing
Jokes
Legal
Money
Movies
Music
Pets
Photo
Science
Software
Source Code
Spirit
Sports
Travel
TV
Writing
Tags:
Syntax Highlighting:
None
Bash
C
C#
C++
CSS
HTML
JSON
Java
JavaScript
Lua
Markdown (PRO members only)
Objective C
PHP
Perl
Python
Ruby
Swift
4CS
6502 ACME Cross Assembler
6502 Kick Assembler
6502 TASM/64TASS
ABAP
AIMMS
ALGOL 68
APT Sources
ARM
ASM (NASM)
ASP
ActionScript
ActionScript 3
Ada
Apache Log
AppleScript
Arduino
Asymptote
AutoIt
Autohotkey
Avisynth
Awk
BASCOM AVR
BNF
BOO
Bash
Basic4GL
Batch
BibTeX
Blitz Basic
Blitz3D
BlitzMax
BrainFuck
C
C (WinAPI)
C Intermediate Language
C for Macs
C#
C++
C++ (WinAPI)
C++ (with Qt extensions)
C: Loadrunner
CAD DCL
CAD Lisp
CFDG
CMake
COBOL
CSS
Ceylon
ChaiScript
Chapel
Clojure
Clone C
Clone C++
CoffeeScript
ColdFusion
Cuesheet
D
DCL
DCPU-16
DCS
DIV
DOT
Dart
Delphi
Delphi Prism (Oxygene)
Diff
E
ECMAScript
EPC
Easytrieve
Eiffel
Email
Erlang
Euphoria
F#
FO Language
Falcon
Filemaker
Formula One
Fortran
FreeBasic
FreeSWITCH
GAMBAS
GDB
GDScript
Game Maker
Genero
Genie
GetText
Go
Godot GLSL
Groovy
GwBasic
HQ9 Plus
HTML
HTML 5
Haskell
Haxe
HicEst
IDL
INI file
INTERCAL
IO
ISPF Panel Definition
Icon
Inno Script
J
JCL
JSON
Java
Java 5
JavaScript
Julia
KSP (Kontakt Script)
KiXtart
Kotlin
LDIF
LLVM
LOL Code
LScript
Latex
Liberty BASIC
Linden Scripting
Lisp
Loco Basic
Logtalk
Lotus Formulas
Lotus Script
Lua
M68000 Assembler
MIX Assembler
MK-61/52
MPASM
MXML
MagikSF
Make
MapBasic
Markdown (PRO members only)
MatLab
Mercury
MetaPost
Modula 2
Modula 3
Motorola 68000 HiSoft Dev
MySQL
Nagios
NetRexx
Nginx
Nim
NullSoft Installer
OCaml
OCaml Brief
Oberon 2
Objeck Programming Langua
Objective C
Octave
Open Object Rexx
OpenBSD PACKET FILTER
OpenGL Shading
Openoffice BASIC
Oracle 11
Oracle 8
Oz
PARI/GP
PCRE
PHP
PHP Brief
PL/I
PL/SQL
POV-Ray
ParaSail
Pascal
Pawn
Per
Perl
Perl 6
Phix
Pic 16
Pike
Pixel Bender
PostScript
PostgreSQL
PowerBuilder
PowerShell
ProFTPd
Progress
Prolog
Properties
ProvideX
Puppet
PureBasic
PyCon
Python
Python for S60
QBasic
QML
R
RBScript
REBOL
REG
RPM Spec
Racket
Rails
Rexx
Robots
Roff Manpage
Ruby
Ruby Gnuplot
Rust
SAS
SCL
SPARK
SPARQL
SQF
SQL
SSH Config
Scala
Scheme
Scilab
SdlBasic
Smalltalk
Smarty
StandardML
StoneScript
SuperCollider
Swift
SystemVerilog
T-SQL
TCL
TeXgraph
Tera Term
TypeScript
TypoScript
UPC
Unicon
UnrealScript
Urbi
VB.NET
VBScript
VHDL
VIM
Vala
Vedit
VeriLog
Visual Pro Log
VisualBasic
VisualFoxPro
WHOIS
WhiteSpace
Winbatch
XBasic
XML
XPP
Xojo
Xorg Config
YAML
YARA
Z80 Assembler
ZXBasic
autoconf
jQuery
mIRC
newLISP
q/kdb+
thinBasic
Paste Expiration:
Never
Burn after read
10 Minutes
1 Hour
1 Day
1 Week
2 Weeks
1 Month
6 Months
1 Year
Paste Exposure:
Public
Unlisted
Private
Folder:
(members only)
Password
NEW
Enabled
Disabled
Burn after read
NEW
Paste Name / Title:
Create New Paste
Hello
Guest
Sign Up
or
Login
Sign in with Facebook
Sign in with Twitter
Sign in with Google
You are currently not logged in, this means you can not edit or delete anything you paste.
Sign Up
or
Login
Public Pastes
Documents
5 min ago | 0.43 KB
Crypto admin access
5 min ago | 0.43 KB
+12,000$ in 2 days
CSS | 5 min ago | 0.72 KB
Free Crypto Method
CSS | 6 min ago | 0.72 KB
Untitled
22 min ago | 12.67 KB
CC:T GPS Beacon
10 hours ago | 0.49 KB
R4 Moon SPI Screen With Wifi
Arduino | 13 hours ago | 13.55 KB
Art school
1 day ago | 1.06 KB
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the
Cookies Policy
.
OK, I Understand
Not a member of Pastebin yet?
Sign Up
, it unlocks many cool features!
