API tools faq
paste
bra_fsn_hu

elastic 6.2.1 queued task grows

bra_fsn_hu
Feb 17th, 2018
88
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 324.04 KB | None | 0 0
raw download clone embed print report
  1. 2018-02-17 12:43:09
  2.  
  3. Full thread dump OpenJDK 64-Bit Server VM (25.144-b01 mixed mode):
  4.  
  5.  
  6.  
  7. "SIGTERM handler" "
  8. #7973 "
  9. daemon "
  10. prio=9 "
  11. os_prio=15 "
  12. tid=0x0000000ae8eb4800 "
  13. nid=0x1936e "
  14. waiting for monitor entry "
  15. [0x00007fffd5f5d000]
  16.  
  17. java.lang.Thread.State: BLOCKED (on object monitor)
  18.  
  19. at java.lang.Shutdown.exit(Shutdown.java:212)
  20.  
  21. - waiting to lock <0x0000000893fed6a8> "
  22. (a java.lang.Class for java.lang.Shutdown)
  23.  
  24. at java.lang.Terminator$1.handle(Terminator.java:52)
  25.  
  26. at sun.misc.Signal$1.run(Signal.java:212)
  27.  
  28. at java.lang.Thread.run(Thread.java:748)
  29.  
  30.  
  31.  
  32. "indices_shutdown[T#5]" "
  33. #7972 "
  34. daemon "
  35. prio=5 "
  36. os_prio=15 "
  37. tid=0x0000000afb69f000 "
  38. nid=0x192ff "
  39. waiting on condition "
  40. [0x00007fffd3836000]
  41.  
  42. java.lang.Thread.State: WAITING (parking)
  43.  
  44. at sun.misc.Unsafe.park(Native Method)
  45.  
  46. - parking to wait for <0x000000086c74aec0> (a java.util.concu\
  47. rrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  48.  
  49. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  50. a:175)
  51.  
  52. at java.util.concurrent.locks.AbstractQueuedSynchronizer$Condi\
  53. tionObject.await(AbstractQueuedSynchronizer.java:2039)
  54.  
  55. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockin\
  56. gQueue.java:442)
  57.  
  58. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  59. xecutor.java:1074)
  60.  
  61. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  62. lExecutor.java:1134)
  63.  
  64. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  65. olExecutor.java:624)
  66.  
  67. at java.lang.Thread.run(Thread.java:748)
  68.  
  69.  
  70.  
  71. "indices_shutdown[T#4]" "
  72. #7971 "
  73. daemon "
  74. prio=5 "
  75. os_prio=15 "
  76. tid=0x0000000afb69e000 "
  77. nid=0x1923a "
  78. waiting on condition "
  79. [0x00007fffd3937000]
  80.  
  81. java.lang.Thread.State: WAITING (parking)
  82.  
  83. at sun.misc.Unsafe.park(Native Method)
  84.  
  85. - parking to wait for <0x000000086c74aec0> (a java.util.concu\
  86. rrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  87.  
  88. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  89. a:175)
  90.  
  91. at java.util.concurrent.locks.AbstractQueuedSynchronizer$Condi\
  92. tionObject.await(AbstractQueuedSynchronizer.java:2039)
  93.  
  94. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockin\
  95. gQueue.java:442)
  96.  
  97. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  98. xecutor.java:1074)
  99.  
  100. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  101. lExecutor.java:1134)
  102.  
  103. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  104. olExecutor.java:624)
  105.  
  106. at java.lang.Thread.run(Thread.java:748)
  107.  
  108.  
  109.  
  110. "indices_shutdown[T#3]" "
  111. #7970 "
  112. daemon "
  113. prio=5 "
  114. os_prio=15 "
  115. tid=0x0000000afb67c800 "
  116. nid=0x191ae "
  117. waiting on condition "
  118. [0x00007fffdadab000]
  119.  
  120. java.lang.Thread.State: WAITING (parking)
  121.  
  122. at sun.misc.Unsafe.park(Native Method)
  123.  
  124. - parking to wait for <0x00000008b315d138> (a java.util.concu\
  125. rrent.locks.ReentrantReadWriteLock$NonfairSync)
  126.  
  127. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  128. a:175)
  129.  
  130. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  131. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  132.  
  133. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  134. reQueued(AbstractQueuedSynchronizer.java:870)
  135.  
  136. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  137. re(AbstractQueuedSynchronizer.java:1199)
  138.  
  139. at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock\
  140. .lock(ReentrantReadWriteLock.java:943)
  141.  
  142. at org.elasticsearch.common.util.concurrent.ReleasableLock.acq\
  143. uire(ReleasableLock.java:55)
  144.  
  145. at org.elasticsearch.index.engine.Engine.flushAndClose(Engine.\
  146. java:1348)
  147.  
  148. at org.elasticsearch.index.shard.IndexShard.close(IndexShard.j\
  149. ava:1193)
  150.  
  151. - locked <0x00000008b3997758> "
  152. (a java.lang.Object)
  153.  
  154. at org.elasticsearch.index.IndexService.closeShard(IndexServic\
  155. e.java:431)
  156.  
  157. at org.elasticsearch.index.IndexService.removeShard(IndexServi\
  158. ce.java:414)
  159.  
  160. - locked <0x00000008a3893210> "
  161. (a org.elasticsearch.index.IndexService)
  162.  
  163. at org.elasticsearch.index.IndexService.close(IndexService.jav\
  164. a:274)
  165.  
  166. - locked <0x00000008a3893210> "
  167. (a org.elasticsearch.index.IndexService)
  168.  
  169. at org.elasticsearch.indices.IndicesService.removeIndex(Indice\
  170. sService.java:554)
  171.  
  172. at org.elasticsearch.indices.IndicesService.lambda$doStop$2(In\
  173. dicesService.java:228)
  174.  
  175. at org.elasticsearch.indices.IndicesService$$Lambda$2254/12644\
  176. 58844.run(Unknown Source)
  177.  
  178. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  179. lExecutor.java:1149)
  180.  
  181. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  182. olExecutor.java:624)
  183.  
  184. at java.lang.Thread.run(Thread.java:748)
  185.  
  186.  
  187.  
  188. "indices_shutdown[T#2]" "
  189. #7969 "
  190. daemon "
  191. prio=5 "
  192. os_prio=15 "
  193. tid=0x0000000afb67b800 "
  194. nid=0x191a9 "
  195. waiting on condition "
  196. [0x00007fffd3d3b000]
  197.  
  198. java.lang.Thread.State: WAITING (parking)
  199.  
  200. at sun.misc.Unsafe.park(Native Method)
  201.  
  202. - parking to wait for <0x000000086c74aec0> (a java.util.concu\
  203. rrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  204.  
  205. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  206. a:175)
  207.  
  208. at java.util.concurrent.locks.AbstractQueuedSynchronizer$Condi\
  209. tionObject.await(AbstractQueuedSynchronizer.java:2039)
  210.  
  211. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockin\
  212. gQueue.java:442)
  213.  
  214. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  215. xecutor.java:1074)
  216.  
  217. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  218. lExecutor.java:1134)
  219.  
  220. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  221. olExecutor.java:624)
  222.  
  223. at java.lang.Thread.run(Thread.java:748)
  224.  
  225.  
  226.  
  227. "indices_shutdown[T#1]" "
  228. #7968 "
  229. daemon "
  230. prio=5 "
  231. os_prio=15 "
  232. tid=0x0000000afb67e000 "
  233. nid=0x1918d "
  234. waiting on condition "
  235. [0x00007fffdb4b2000]
  236.  
  237. java.lang.Thread.State: WAITING (parking)
  238.  
  239. at sun.misc.Unsafe.park(Native Method)
  240.  
  241. - parking to wait for <0x000000086c74aec0> (a java.util.concu\
  242. rrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  243.  
  244. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  245. a:175)
  246.  
  247. at java.util.concurrent.locks.AbstractQueuedSynchronizer$Condi\
  248. tionObject.await(AbstractQueuedSynchronizer.java:2039)
  249.  
  250. at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockin\
  251. gQueue.java:442)
  252.  
  253. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  254. xecutor.java:1074)
  255.  
  256. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  257. lExecutor.java:1134)
  258.  
  259. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  260. olExecutor.java:624)
  261.  
  262. at java.lang.Thread.run(Thread.java:748)
  263.  
  264.  
  265.  
  266. "elasticsearch[fmfe16][generic][T#154]" "
  267. #7964 "
  268. daemon "
  269. prio=5 "
  270. os_prio=15 "
  271. tid=0x0000000ae9b1a000 "
  272. nid=0x1912d "
  273. waiting on condition "
  274. [0x00007fffd1e1c000]
  275.  
  276. java.lang.Thread.State: WAITING (parking)
  277.  
  278. at sun.misc.Unsafe.park(Native Method)
  279.  
  280. - parking to wait for <0x0000000872bdf1e0> (a org.elasticsear\
  281. ch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
  282.  
  283. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  284. a:175)
  285.  
  286. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  287. ransferQueue.java:737)
  288.  
  289. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  290. rQueue.java:647)
  291.  
  292. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  293. rQueue.java:1269)
  294.  
  295. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  296. xecutor.java:1074)
  297.  
  298. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  299. lExecutor.java:1134)
  300.  
  301. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  302. olExecutor.java:624)
  303.  
  304. at java.lang.Thread.run(Thread.java:748)
  305.  
  306.  
  307.  
  308. "elasticsearch[fmfe16][generic][T#95]" "
  309. #7913 "
  310. daemon "
  311. prio=5 "
  312. os_prio=15 "
  313. tid=0x0000000b10ecc000 "
  314. nid=0x194e2 "
  315. waiting on condition "
  316. [0x00007fffd6664000]
  317.  
  318. java.lang.Thread.State: WAITING (parking)
  319.  
  320. at sun.misc.Unsafe.park(Native Method)
  321.  
  322. - parking to wait for <0x0000000872bdf1e0> (a org.elasticsear\
  323. ch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
  324.  
  325. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  326. a:175)
  327.  
  328. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  329. ransferQueue.java:737)
  330.  
  331. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  332. rQueue.java:647)
  333.  
  334. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  335. rQueue.java:1269)
  336.  
  337. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  338. xecutor.java:1074)
  339.  
  340. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  341. lExecutor.java:1134)
  342.  
  343. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  344. olExecutor.java:624)
  345.  
  346. at java.lang.Thread.run(Thread.java:748)
  347.  
  348.  
  349.  
  350. "elasticsearch[fmfe16][generic][T#86]" "
  351. #7883 "
  352. daemon "
  353. prio=5 "
  354. os_prio=15 "
  355. tid=0x0000000b00aaa800 "
  356. nid=0x1943e "
  357. waiting on condition "
  358. [0x00007fffca6a5000]
  359.  
  360. java.lang.Thread.State: WAITING (parking)
  361.  
  362. at sun.misc.Unsafe.park(Native Method)
  363.  
  364. - parking to wait for <0x0000000872bdf1e0> (a org.elasticsear\
  365. ch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
  366.  
  367. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  368. a:175)
  369.  
  370. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  371. ransferQueue.java:737)
  372.  
  373. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  374. rQueue.java:647)
  375.  
  376. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  377. rQueue.java:1269)
  378.  
  379. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  380. xecutor.java:1074)
  381.  
  382. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  383. lExecutor.java:1134)
  384.  
  385. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  386. olExecutor.java:624)
  387.  
  388. at java.lang.Thread.run(Thread.java:748)
  389.  
  390.  
  391.  
  392. "Thread-2" "
  393. #22 "
  394. prio=5 "
  395. os_prio=15 "
  396. tid=0x0000000ae9178000 "
  397. nid=0x19b22 "
  398. waiting on condition "
  399. [0x00007fffc6160000]
  400.  
  401. java.lang.Thread.State: TIMED_WAITING (parking)
  402.  
  403. at sun.misc.Unsafe.park(Native Method)
  404.  
  405. - parking to wait for <0x00000008698d8678> (a java.util.concu\
  406. rrent.CountDownLatch$Sync)
  407.  
  408. at java.util.concurrent.locks.LockSupport.parkNanos(LockSuppor\
  409. t.java:215)
  410.  
  411. at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcq\
  412. uireSharedNanos(AbstractQueuedSynchronizer.java:1037)
  413.  
  414. at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAc\
  415. quireSharedNanos(AbstractQueuedSynchronizer.java:1328)
  416.  
  417. at java.util.concurrent.CountDownLatch.await(CountDownLatch.ja\
  418. va:277)
  419.  
  420. at org.elasticsearch.indices.IndicesService.doStop(IndicesServ\
  421. ice.java:235)
  422.  
  423. at org.elasticsearch.common.component.AbstractLifecycleCompone\
  424. nt.stop(AbstractLifecycleComponent.java:85)
  425.  
  426. at org.elasticsearch.node.Node.stop(Node.java:740)
  427.  
  428. at org.elasticsearch.node.Node.close(Node.java:752)
  429.  
  430. - locked <0x0000000872bf3560> "
  431. (a org.elasticsearch.bootstrap.Bootstrap$5)
  432.  
  433. at org.apache.lucene.util.IOUtils.close(IOUtils.java:89)
  434.  
  435. at org.apache.lucene.util.IOUtils.close(IOUtils.java:76)
  436.  
  437. at org.elasticsearch.bootstrap.Bootstrap$4.run(Bootstrap.java:\
  438. 186)
  439.  
  440.  
  441.  
  442. "SIGTERM handler" "
  443. #7874 "
  444. daemon "
  445. prio=9 "
  446. os_prio=15 "
  447. tid=0x0000000ae8eb4000 "
  448. nid=0x19419 "
  449. in Object.wait() "
  450. [0x00007fffc8c8b000]
  451.  
  452. java.lang.Thread.State: WAITING (on object monitor)
  453.  
  454. at java.lang.Object.wait(Native Method)
  455.  
  456. at java.lang.Thread.join(Thread.java:1252)
  457.  
  458. - locked <0x0000000893f5f5a8> "
  459. (a org.elasticsearch.bootstrap.Bootstrap$4)
  460.  
  461. at java.lang.Thread.join(Thread.java:1326)
  462.  
  463. at java.lang.ApplicationShutdownHooks.runHooks(ApplicationShut\
  464. downHooks.java:106)
  465.  
  466. at java.lang.ApplicationShutdownHooks$1.run(ApplicationShutdow\
  467. nHooks.java:46)
  468.  
  469. at java.lang.Shutdown.runHooks(Shutdown.java:123)
  470.  
  471. at java.lang.Shutdown.sequence(Shutdown.java:167)
  472.  
  473. at java.lang.Shutdown.exit(Shutdown.java:212)
  474.  
  475. - locked <0x0000000893fed6a8> "
  476. (a java.lang.Class for java.lang.Shutdown)
  477.  
  478. at java.lang.Terminator$1.handle(Terminator.java:52)
  479.  
  480. at sun.misc.Signal$1.run(Signal.java:212)
  481.  
  482. at java.lang.Thread.run(Thread.java:748)
  483.  
  484.  
  485.  
  486. "elasticsearch[fmfe16][fetch_shard_store][T#117]" "
  487. #7849 "
  488. daemon "
  489. prio=5 "
  490. os_prio=15 "
  491. tid=0x0000000b10e54800 "
  492. nid=0x19b78 "
  493. waiting on condition "
  494. [0x00007fffce8e7000]
  495.  
  496. java.lang.Thread.State: WAITING (parking)
  497.  
  498. at sun.misc.Unsafe.park(Native Method)
  499.  
  500. - parking to wait for <0x0000000872bceff0> (a org.elasticsear\
  501. ch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
  502.  
  503. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  504. a:175)
  505.  
  506. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  507. ransferQueue.java:737)
  508.  
  509. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  510. rQueue.java:647)
  511.  
  512. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  513. rQueue.java:1269)
  514.  
  515. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  516. xecutor.java:1074)
  517.  
  518. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  519. lExecutor.java:1134)
  520.  
  521. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  522. olExecutor.java:624)
  523.  
  524. at java.lang.Thread.run(Thread.java:748)
  525.  
  526.  
  527.  
  528. "elasticsearch[fmfe16][listener][T#10]" "
  529. #2132 "
  530. daemon "
  531. prio=5 "
  532. os_prio=15 "
  533. tid=0x0000000b03910800 "
  534. nid=0x19396 "
  535. waiting on condition "
  536. [0x00007fffd908e000]
  537.  
  538. java.lang.Thread.State: WAITING (parking)
  539.  
  540. at sun.misc.Unsafe.park(Native Method)
  541.  
  542. - parking to wait for <0x000000087418be38> (a java.util.concu\
  543. rrent.LinkedTransferQueue)
  544.  
  545. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  546. a:175)
  547.  
  548. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  549. ransferQueue.java:737)
  550.  
  551. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  552. rQueue.java:647)
  553.  
  554. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  555. rQueue.java:1269)
  556.  
  557. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  558. xecutor.java:1074)
  559.  
  560. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  561. lExecutor.java:1134)
  562.  
  563. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  564. olExecutor.java:624)
  565.  
  566. at java.lang.Thread.run(Thread.java:748)
  567.  
  568.  
  569.  
  570. "elasticsearch[fmfe16][listener][T#9]" "
  571. #2127 "
  572. daemon "
  573. prio=5 "
  574. os_prio=15 "
  575. tid=0x0000000813838800 "
  576. nid=0x19267 "
  577. waiting on condition "
  578. [0x00007fffc9f9e000]
  579.  
  580. java.lang.Thread.State: WAITING (parking)
  581.  
  582. at sun.misc.Unsafe.park(Native Method)
  583.  
  584. - parking to wait for <0x000000087418be38> (a java.util.concu\
  585. rrent.LinkedTransferQueue)
  586.  
  587. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  588. a:175)
  589.  
  590. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  591. ransferQueue.java:737)
  592.  
  593. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  594. rQueue.java:647)
  595.  
  596. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  597. rQueue.java:1269)
  598.  
  599. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  600. xecutor.java:1074)
  601.  
  602. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  603. lExecutor.java:1134)
  604.  
  605. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  606. olExecutor.java:624)
  607.  
  608. at java.lang.Thread.run(Thread.java:748)
  609.  
  610.  
  611.  
  612. "elasticsearch[fmfe16][listener][T#8]" "
  613. #2120 "
  614. daemon "
  615. prio=5 "
  616. os_prio=15 "
  617. tid=0x0000000b04f34000 "
  618. nid=0x18dc9 "
  619. waiting on condition "
  620. [0x00007fffcb8b7000]
  621.  
  622. java.lang.Thread.State: WAITING (parking)
  623.  
  624. at sun.misc.Unsafe.park(Native Method)
  625.  
  626. - parking to wait for <0x000000087418be38> (a java.util.concu\
  627. rrent.LinkedTransferQueue)
  628.  
  629. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  630. a:175)
  631.  
  632. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  633. ransferQueue.java:737)
  634.  
  635. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  636. rQueue.java:647)
  637.  
  638. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  639. rQueue.java:1269)
  640.  
  641. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  642. xecutor.java:1074)
  643.  
  644. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  645. lExecutor.java:1134)
  646.  
  647. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  648. olExecutor.java:624)
  649.  
  650. at java.lang.Thread.run(Thread.java:748)
  651.  
  652.  
  653.  
  654. "elasticsearch[fmfe16][listener][T#7]" "
  655. #2103 "
  656. daemon "
  657. prio=5 "
  658. os_prio=15 "
  659. tid=0x0000000b11264000 "
  660. nid=0x193f8 "
  661. waiting on condition "
  662. [0x00007fffc7776000]
  663.  
  664. java.lang.Thread.State: WAITING (parking)
  665.  
  666. at sun.misc.Unsafe.park(Native Method)
  667.  
  668. - parking to wait for <0x000000087418be38> (a java.util.concu\
  669. rrent.LinkedTransferQueue)
  670.  
  671. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  672. a:175)
  673.  
  674. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  675. ransferQueue.java:737)
  676.  
  677. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  678. rQueue.java:647)
  679.  
  680. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  681. rQueue.java:1269)
  682.  
  683. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  684. xecutor.java:1074)
  685.  
  686. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  687. lExecutor.java:1134)
  688.  
  689. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  690. olExecutor.java:624)
  691.  
  692. at java.lang.Thread.run(Thread.java:748)
  693.  
  694.  
  695.  
  696. "elasticsearch[fmfe16][listener][T#6]" "
  697. #2095 "
  698. daemon "
  699. prio=5 "
  700. os_prio=15 "
  701. tid=0x0000000b11263000 "
  702. nid=0x187cf "
  703. waiting on condition "
  704. [0x00007fffc807f000]
  705.  
  706. java.lang.Thread.State: WAITING (parking)
  707.  
  708. at sun.misc.Unsafe.park(Native Method)
  709.  
  710. - parking to wait for <0x000000087418be38> (a java.util.concu\
  711. rrent.LinkedTransferQueue)
  712.  
  713. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  714. a:175)
  715.  
  716. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  717. ransferQueue.java:737)
  718.  
  719. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  720. rQueue.java:647)
  721.  
  722. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  723. rQueue.java:1269)
  724.  
  725. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  726. xecutor.java:1074)
  727.  
  728. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  729. lExecutor.java:1134)
  730.  
  731. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  732. olExecutor.java:624)
  733.  
  734. at java.lang.Thread.run(Thread.java:748)
  735.  
  736.  
  737.  
  738. "elasticsearch[fmfe16][listener][T#5]" "
  739. #2075 "
  740. daemon "
  741. prio=5 "
  742. os_prio=15 "
  743. tid=0x0000000afe0f4000 "
  744. nid=0x188b8 "
  745. waiting on condition "
  746. [0x00007fffc9e9d000]
  747.  
  748. java.lang.Thread.State: WAITING (parking)
  749.  
  750. at sun.misc.Unsafe.park(Native Method)
  751.  
  752. - parking to wait for <0x000000087418be38> (a java.util.concu\
  753. rrent.LinkedTransferQueue)
  754.  
  755. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  756. a:175)
  757.  
  758. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  759. ransferQueue.java:737)
  760.  
  761. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  762. rQueue.java:647)
  763.  
  764. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  765. rQueue.java:1269)
  766.  
  767. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  768. xecutor.java:1074)
  769.  
  770. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  771. lExecutor.java:1134)
  772.  
  773. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  774. olExecutor.java:624)
  775.  
  776. at java.lang.Thread.run(Thread.java:748)
  777.  
  778.  
  779.  
  780. "elasticsearch[fmfe16][listener][T#4]" "
  781. #2062 "
  782. daemon "
  783. prio=5 "
  784. os_prio=15 "
  785. tid=0x0000000b0c73f000 "
  786. nid=0x1922c "
  787. waiting on condition "
  788. [0x00007fffcb7b6000]
  789.  
  790. java.lang.Thread.State: WAITING (parking)
  791.  
  792. at sun.misc.Unsafe.park(Native Method)
  793.  
  794. - parking to wait for <0x000000087418be38> (a java.util.concu\
  795. rrent.LinkedTransferQueue)
  796.  
  797. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  798. a:175)
  799.  
  800. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  801. ransferQueue.java:737)
  802.  
  803. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  804. rQueue.java:647)
  805.  
  806. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  807. rQueue.java:1269)
  808.  
  809. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  810. xecutor.java:1074)
  811.  
  812. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  813. lExecutor.java:1134)
  814.  
  815. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  816. olExecutor.java:624)
  817.  
  818. at java.lang.Thread.run(Thread.java:748)
  819.  
  820.  
  821.  
  822. "elasticsearch[fmfe16][listener][T#3]" "
  823. #2055 "
  824. daemon "
  825. prio=5 "
  826. os_prio=15 "
  827. tid=0x0000000afe0f2800 "
  828. nid=0x187d4 "
  829. waiting on condition "
  830. [0x00007fffc9897000]
  831.  
  832. java.lang.Thread.State: WAITING (parking)
  833.  
  834. at sun.misc.Unsafe.park(Native Method)
  835.  
  836. - parking to wait for <0x000000087418be38> (a java.util.concu\
  837. rrent.LinkedTransferQueue)
  838.  
  839. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  840. a:175)
  841.  
  842. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  843. ransferQueue.java:737)
  844.  
  845. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  846. rQueue.java:647)
  847.  
  848. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  849. rQueue.java:1269)
  850.  
  851. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  852. xecutor.java:1074)
  853.  
  854. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  855. lExecutor.java:1134)
  856.  
  857. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  858. olExecutor.java:624)
  859.  
  860. at java.lang.Thread.run(Thread.java:748)
  861.  
  862.  
  863.  
  864. "elasticsearch[fmfe16][listener][T#2]" "
  865. #2024 "
  866. daemon "
  867. prio=5 "
  868. os_prio=15 "
  869. tid=0x00000008138d5000 "
  870. nid=0x18ba0 "
  871. waiting on condition "
  872. [0x00007fffd706e000]
  873.  
  874. java.lang.Thread.State: WAITING (parking)
  875.  
  876. at sun.misc.Unsafe.park(Native Method)
  877.  
  878. - parking to wait for <0x000000087418be38> (a java.util.concu\
  879. rrent.LinkedTransferQueue)
  880.  
  881. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  882. a:175)
  883.  
  884. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  885. ransferQueue.java:737)
  886.  
  887. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  888. rQueue.java:647)
  889.  
  890. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  891. rQueue.java:1269)
  892.  
  893. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  894. xecutor.java:1074)
  895.  
  896. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  897. lExecutor.java:1134)
  898.  
  899. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  900. olExecutor.java:624)
  901.  
  902. at java.lang.Thread.run(Thread.java:748)
  903.  
  904.  
  905.  
  906. "elasticsearch[fmfe16][listener][T#1]" "
  907. #2004 "
  908. daemon "
  909. prio=5 "
  910. os_prio=15 "
  911. tid=0x00000008138d9000 "
  912. nid=0x18dff "
  913. waiting on condition "
  914. [0x00007fffd8e8c000]
  915.  
  916. java.lang.Thread.State: WAITING (parking)
  917.  
  918. at sun.misc.Unsafe.park(Native Method)
  919.  
  920. - parking to wait for <0x000000087418be38> (a java.util.concu\
  921. rrent.LinkedTransferQueue)
  922.  
  923. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  924. a:175)
  925.  
  926. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  927. ransferQueue.java:737)
  928.  
  929. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  930. rQueue.java:647)
  931.  
  932. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  933. rQueue.java:1269)
  934.  
  935. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  936. xecutor.java:1074)
  937.  
  938. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  939. lExecutor.java:1134)
  940.  
  941. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  942. olExecutor.java:624)
  943.  
  944. at java.lang.Thread.run(Thread.java:748)
  945.  
  946.  
  947.  
  948. "elasticsearch[fmfe16][index][T#24]" "
  949. #436 "
  950. daemon "
  951. prio=5 "
  952. os_prio=15 "
  953. tid=0x0000000ae8ac9000 "
  954. nid=0x18ef3 "
  955. waiting on condition "
  956. [0x00007fffd9290000]
  957.  
  958. java.lang.Thread.State: WAITING (parking)
  959.  
  960. at sun.misc.Unsafe.park(Native Method)
  961.  
  962. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  963. rrent.locks.ReentrantLock$NonfairSync)
  964.  
  965. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  966. a:175)
  967.  
  968. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  969. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  970.  
  971. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  972. reQueued(AbstractQueuedSynchronizer.java:870)
  973.  
  974. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  975. re(AbstractQueuedSynchronizer.java:1199)
  976.  
  977. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  978. eentrantLock.java:209)
  979.  
  980. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  981. .java:285)
  982.  
  983. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  984. KeyedLock.java:76)
  985.  
  986. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  987. iveVersionMap.java:431)
  988.  
  989. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  990. ombstones(InternalEngine.java:1661)
  991.  
  992. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  993. etedTombstones(InternalEngine.java:1348)
  994.  
  995. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  996. nalEngine.java:1427)
  997.  
  998. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  999. ngine.java:637)
  1000.  
  1001. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1002. a:850)
  1003.  
  1004. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1005. etService.java:153)
  1006.  
  1007. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1008. vice.java:81)
  1009.  
  1010. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1011. Helper.java:74)
  1012.  
  1013. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1014. Operation(TransportUpdateAction.java:174)
  1015.  
  1016. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1017. Operation(TransportUpdateAction.java:167)
  1018.  
  1019. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1020. Operation(TransportUpdateAction.java:67)
  1021.  
  1022. at org.elasticsearch.action.support.single.instance.TransportI\
  1023. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1024. nsportInstanceSingleOperationAction.java:243)
  1025.  
  1026. at org.elasticsearch.action.support.single.instance.TransportI\
  1027. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1028. nsportInstanceSingleOperationAction.java:239)
  1029.  
  1030. at org.elasticsearch.transport.TransportRequestHandler.message\
  1031. Received(TransportRequestHandler.java:30)
  1032.  
  1033. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1034. essageReceived(RequestHandlerRegistry.java:66)
  1035.  
  1036. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1037. un(TcpTransport.java:1555)
  1038.  
  1039. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1040. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1041.  
  1042. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1043. un(AbstractRunnable.java:37)
  1044.  
  1045. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1046. lExecutor.java:1149)
  1047.  
  1048. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1049. olExecutor.java:624)
  1050.  
  1051. at java.lang.Thread.run(Thread.java:748)
  1052.  
  1053.  
  1054.  
  1055. "elasticsearch[fmfe16][index][T#23]" "
  1056. #435 "
  1057. daemon "
  1058. prio=5 "
  1059. os_prio=15 "
  1060. tid=0x0000000ae93ee000 "
  1061. nid=0x18eaa "
  1062. waiting on condition "
  1063. [0x00007fffd9c9a000]
  1064.  
  1065. java.lang.Thread.State: WAITING (parking)
  1066.  
  1067. at sun.misc.Unsafe.park(Native Method)
  1068.  
  1069. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1070. rrent.locks.ReentrantLock$NonfairSync)
  1071.  
  1072. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1073. a:175)
  1074.  
  1075. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1076. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1077.  
  1078. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1079. reQueued(AbstractQueuedSynchronizer.java:870)
  1080.  
  1081. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1082. re(AbstractQueuedSynchronizer.java:1199)
  1083.  
  1084. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1085. eentrantLock.java:209)
  1086.  
  1087. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1088. .java:285)
  1089.  
  1090. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1091. KeyedLock.java:76)
  1092.  
  1093. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1094. iveVersionMap.java:431)
  1095.  
  1096. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1097. ombstones(InternalEngine.java:1661)
  1098.  
  1099. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1100. etedTombstones(InternalEngine.java:1348)
  1101.  
  1102. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1103. nalEngine.java:1427)
  1104.  
  1105. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1106. ngine.java:637)
  1107.  
  1108. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1109. a:850)
  1110.  
  1111. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1112. etService.java:153)
  1113.  
  1114. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1115. vice.java:81)
  1116.  
  1117. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1118. Helper.java:74)
  1119.  
  1120. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1121. Operation(TransportUpdateAction.java:174)
  1122.  
  1123. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1124. Operation(TransportUpdateAction.java:167)
  1125.  
  1126. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1127. Operation(TransportUpdateAction.java:67)
  1128.  
  1129. at org.elasticsearch.action.support.single.instance.TransportI\
  1130. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1131. nsportInstanceSingleOperationAction.java:243)
  1132.  
  1133. at org.elasticsearch.action.support.single.instance.TransportI\
  1134. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1135. nsportInstanceSingleOperationAction.java:239)
  1136.  
  1137. at org.elasticsearch.transport.TransportRequestHandler.message\
  1138. Received(TransportRequestHandler.java:30)
  1139.  
  1140. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1141. essageReceived(RequestHandlerRegistry.java:66)
  1142.  
  1143. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1144. un(TcpTransport.java:1555)
  1145.  
  1146. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1147. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1148.  
  1149. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1150. un(AbstractRunnable.java:37)
  1151.  
  1152. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1153. lExecutor.java:1149)
  1154.  
  1155. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1156. olExecutor.java:624)
  1157.  
  1158. at java.lang.Thread.run(Thread.java:748)
  1159.  
  1160.  
  1161.  
  1162. "elasticsearch[fmfe16][index][T#22]" "
  1163. #434 "
  1164. daemon "
  1165. prio=5 "
  1166. os_prio=15 "
  1167. tid=0x0000000ae8cd0000 "
  1168. nid=0x18e1d "
  1169. waiting on condition "
  1170. [0x00007fffd9391000]
  1171.  
  1172. java.lang.Thread.State: WAITING (parking)
  1173.  
  1174. at sun.misc.Unsafe.park(Native Method)
  1175.  
  1176. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1177. rrent.locks.ReentrantLock$NonfairSync)
  1178.  
  1179. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1180. a:175)
  1181.  
  1182. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1183. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1184.  
  1185. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1186. reQueued(AbstractQueuedSynchronizer.java:870)
  1187.  
  1188. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1189. re(AbstractQueuedSynchronizer.java:1199)
  1190.  
  1191. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1192. eentrantLock.java:209)
  1193.  
  1194. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1195. .java:285)
  1196.  
  1197. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1198. KeyedLock.java:76)
  1199.  
  1200. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1201. iveVersionMap.java:431)
  1202.  
  1203. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1204. ombstones(InternalEngine.java:1661)
  1205.  
  1206. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1207. etedTombstones(InternalEngine.java:1348)
  1208.  
  1209. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1210. nalEngine.java:1427)
  1211.  
  1212. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1213. ngine.java:637)
  1214.  
  1215. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1216. a:850)
  1217.  
  1218. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1219. etService.java:153)
  1220.  
  1221. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1222. vice.java:81)
  1223.  
  1224. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1225. Helper.java:74)
  1226.  
  1227. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1228. Operation(TransportUpdateAction.java:174)
  1229.  
  1230. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1231. Operation(TransportUpdateAction.java:167)
  1232.  
  1233. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1234. Operation(TransportUpdateAction.java:67)
  1235.  
  1236. at org.elasticsearch.action.support.single.instance.TransportI\
  1237. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1238. nsportInstanceSingleOperationAction.java:243)
  1239.  
  1240. at org.elasticsearch.action.support.single.instance.TransportI\
  1241. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1242. nsportInstanceSingleOperationAction.java:239)
  1243.  
  1244. at org.elasticsearch.transport.TransportRequestHandler.message\
  1245. Received(TransportRequestHandler.java:30)
  1246.  
  1247. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1248. essageReceived(RequestHandlerRegistry.java:66)
  1249.  
  1250. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1251. un(TcpTransport.java:1555)
  1252.  
  1253. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1254. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1255.  
  1256. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1257. un(AbstractRunnable.java:37)
  1258.  
  1259. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1260. lExecutor.java:1149)
  1261.  
  1262. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1263. olExecutor.java:624)
  1264.  
  1265. at java.lang.Thread.run(Thread.java:748)
  1266.  
  1267.  
  1268.  
  1269. "elasticsearch[fmfe16][index][T#21]" "
  1270. #432 "
  1271. daemon "
  1272. prio=5 "
  1273. os_prio=15 "
  1274. tid=0x0000000afde8c000 "
  1275. nid=0x189f6 "
  1276. waiting on condition "
  1277. [0x00007fffd6d6b000]
  1278.  
  1279. java.lang.Thread.State: WAITING (parking)
  1280.  
  1281. at sun.misc.Unsafe.park(Native Method)
  1282.  
  1283. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1284. rrent.locks.ReentrantLock$NonfairSync)
  1285.  
  1286. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1287. a:175)
  1288.  
  1289. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1290. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1291.  
  1292. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1293. reQueued(AbstractQueuedSynchronizer.java:870)
  1294.  
  1295. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1296. re(AbstractQueuedSynchronizer.java:1199)
  1297.  
  1298. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1299. eentrantLock.java:209)
  1300.  
  1301. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1302. .java:285)
  1303.  
  1304. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1305. KeyedLock.java:76)
  1306.  
  1307. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1308. iveVersionMap.java:431)
  1309.  
  1310. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1311. ombstones(InternalEngine.java:1661)
  1312.  
  1313. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1314. etedTombstones(InternalEngine.java:1348)
  1315.  
  1316. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1317. nalEngine.java:1427)
  1318.  
  1319. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1320. ngine.java:637)
  1321.  
  1322. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1323. a:850)
  1324.  
  1325. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1326. etService.java:153)
  1327.  
  1328. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1329. vice.java:81)
  1330.  
  1331. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1332. Helper.java:74)
  1333.  
  1334. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1335. Operation(TransportUpdateAction.java:174)
  1336.  
  1337. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1338. Operation(TransportUpdateAction.java:167)
  1339.  
  1340. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1341. Operation(TransportUpdateAction.java:67)
  1342.  
  1343. at org.elasticsearch.action.support.single.instance.TransportI\
  1344. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1345. nsportInstanceSingleOperationAction.java:243)
  1346.  
  1347. at org.elasticsearch.action.support.single.instance.TransportI\
  1348. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1349. nsportInstanceSingleOperationAction.java:239)
  1350.  
  1351. at org.elasticsearch.transport.TransportRequestHandler.message\
  1352. Received(TransportRequestHandler.java:30)
  1353.  
  1354. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1355. essageReceived(RequestHandlerRegistry.java:66)
  1356.  
  1357. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1358. un(TcpTransport.java:1555)
  1359.  
  1360. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1361. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1362.  
  1363. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1364. un(AbstractRunnable.java:37)
  1365.  
  1366. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1367. lExecutor.java:1149)
  1368.  
  1369. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1370. olExecutor.java:624)
  1371.  
  1372. at java.lang.Thread.run(Thread.java:748)
  1373.  
  1374.  
  1375.  
  1376. "elasticsearch[fmfe16][index][T#20]" "
  1377. #431 "
  1378. daemon "
  1379. prio=5 "
  1380. os_prio=15 "
  1381. tid=0x0000000ae9ac0800 "
  1382. nid=0x189e4 "
  1383. waiting on condition "
  1384. [0x00007fffd5c5a000]
  1385.  
  1386. java.lang.Thread.State: WAITING (parking)
  1387.  
  1388. at sun.misc.Unsafe.park(Native Method)
  1389.  
  1390. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1391. rrent.locks.ReentrantLock$NonfairSync)
  1392.  
  1393. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1394. a:175)
  1395.  
  1396. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1397. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1398.  
  1399. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1400. reQueued(AbstractQueuedSynchronizer.java:870)
  1401.  
  1402. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1403. re(AbstractQueuedSynchronizer.java:1199)
  1404.  
  1405. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1406. eentrantLock.java:209)
  1407.  
  1408. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1409. .java:285)
  1410.  
  1411. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1412. KeyedLock.java:76)
  1413.  
  1414. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1415. iveVersionMap.java:431)
  1416.  
  1417. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1418. ombstones(InternalEngine.java:1661)
  1419.  
  1420. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1421. etedTombstones(InternalEngine.java:1348)
  1422.  
  1423. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1424. nalEngine.java:1427)
  1425.  
  1426. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1427. ngine.java:637)
  1428.  
  1429. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1430. a:850)
  1431.  
  1432. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1433. etService.java:153)
  1434.  
  1435. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1436. vice.java:81)
  1437.  
  1438. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1439. Helper.java:74)
  1440.  
  1441. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1442. Operation(TransportUpdateAction.java:174)
  1443.  
  1444. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1445. Operation(TransportUpdateAction.java:167)
  1446.  
  1447. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1448. Operation(TransportUpdateAction.java:67)
  1449.  
  1450. at org.elasticsearch.action.support.single.instance.TransportI\
  1451. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1452. nsportInstanceSingleOperationAction.java:243)
  1453.  
  1454. at org.elasticsearch.action.support.single.instance.TransportI\
  1455. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1456. nsportInstanceSingleOperationAction.java:239)
  1457.  
  1458. at org.elasticsearch.transport.TransportRequestHandler.message\
  1459. Received(TransportRequestHandler.java:30)
  1460.  
  1461. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1462. essageReceived(RequestHandlerRegistry.java:66)
  1463.  
  1464. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1465. un(TcpTransport.java:1555)
  1466.  
  1467. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1468. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1469.  
  1470. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1471. un(AbstractRunnable.java:37)
  1472.  
  1473. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1474. lExecutor.java:1149)
  1475.  
  1476. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1477. olExecutor.java:624)
  1478.  
  1479. at java.lang.Thread.run(Thread.java:748)
  1480.  
  1481.  
  1482.  
  1483. "elasticsearch[fmfe16][index][T#19]" "
  1484. #430 "
  1485. daemon "
  1486. prio=5 "
  1487. os_prio=15 "
  1488. tid=0x0000000ae8e1c000 "
  1489. nid=0x189e2 "
  1490. waiting on condition "
  1491. [0x00007fffcaaa9000]
  1492.  
  1493. java.lang.Thread.State: WAITING (parking)
  1494.  
  1495. at sun.misc.Unsafe.park(Native Method)
  1496.  
  1497. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1498. rrent.locks.ReentrantLock$NonfairSync)
  1499.  
  1500. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1501. a:175)
  1502.  
  1503. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1504. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1505.  
  1506. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1507. reQueued(AbstractQueuedSynchronizer.java:870)
  1508.  
  1509. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1510. re(AbstractQueuedSynchronizer.java:1199)
  1511.  
  1512. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1513. eentrantLock.java:209)
  1514.  
  1515. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1516. .java:285)
  1517.  
  1518. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1519. KeyedLock.java:76)
  1520.  
  1521. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1522. iveVersionMap.java:431)
  1523.  
  1524. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1525. ombstones(InternalEngine.java:1661)
  1526.  
  1527. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1528. etedTombstones(InternalEngine.java:1348)
  1529.  
  1530. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1531. nalEngine.java:1427)
  1532.  
  1533. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1534. ngine.java:637)
  1535.  
  1536. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1537. a:850)
  1538.  
  1539. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1540. etService.java:153)
  1541.  
  1542. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1543. vice.java:81)
  1544.  
  1545. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1546. Helper.java:74)
  1547.  
  1548. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1549. Operation(TransportUpdateAction.java:174)
  1550.  
  1551. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1552. Operation(TransportUpdateAction.java:167)
  1553.  
  1554. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1555. Operation(TransportUpdateAction.java:67)
  1556.  
  1557. at org.elasticsearch.action.support.single.instance.TransportI\
  1558. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1559. nsportInstanceSingleOperationAction.java:243)
  1560.  
  1561. at org.elasticsearch.action.support.single.instance.TransportI\
  1562. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1563. nsportInstanceSingleOperationAction.java:239)
  1564.  
  1565. at org.elasticsearch.transport.TransportRequestHandler.message\
  1566. Received(TransportRequestHandler.java:30)
  1567.  
  1568. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1569. essageReceived(RequestHandlerRegistry.java:66)
  1570.  
  1571. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1572. un(TcpTransport.java:1555)
  1573.  
  1574. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1575. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1576.  
  1577. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1578. un(AbstractRunnable.java:37)
  1579.  
  1580. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1581. lExecutor.java:1149)
  1582.  
  1583. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1584. olExecutor.java:624)
  1585.  
  1586. at java.lang.Thread.run(Thread.java:748)
  1587.  
  1588.  
  1589.  
  1590. "elasticsearch[fmfe16][index][T#18]" "
  1591. #429 "
  1592. daemon "
  1593. prio=5 "
  1594. os_prio=15 "
  1595. tid=0x0000000b042ad000 "
  1596. nid=0x18903 "
  1597. waiting on condition "
  1598. [0x00007fffc9392000]
  1599.  
  1600. java.lang.Thread.State: WAITING (parking)
  1601.  
  1602. at sun.misc.Unsafe.park(Native Method)
  1603.  
  1604. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1605. rrent.locks.ReentrantLock$NonfairSync)
  1606.  
  1607. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1608. a:175)
  1609.  
  1610. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1611. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1612.  
  1613. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1614. reQueued(AbstractQueuedSynchronizer.java:870)
  1615.  
  1616. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1617. re(AbstractQueuedSynchronizer.java:1199)
  1618.  
  1619. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1620. eentrantLock.java:209)
  1621.  
  1622. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1623. .java:285)
  1624.  
  1625. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1626. KeyedLock.java:76)
  1627.  
  1628. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1629. iveVersionMap.java:431)
  1630.  
  1631. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1632. ombstones(InternalEngine.java:1661)
  1633.  
  1634. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1635. etedTombstones(InternalEngine.java:1348)
  1636.  
  1637. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1638. nalEngine.java:1427)
  1639.  
  1640. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1641. ngine.java:637)
  1642.  
  1643. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1644. a:850)
  1645.  
  1646. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1647. etService.java:153)
  1648.  
  1649. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1650. vice.java:81)
  1651.  
  1652. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1653. Helper.java:74)
  1654.  
  1655. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1656. Operation(TransportUpdateAction.java:174)
  1657.  
  1658. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1659. Operation(TransportUpdateAction.java:167)
  1660.  
  1661. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1662. Operation(TransportUpdateAction.java:67)
  1663.  
  1664. at org.elasticsearch.action.support.single.instance.TransportI\
  1665. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1666. nsportInstanceSingleOperationAction.java:243)
  1667.  
  1668. at org.elasticsearch.action.support.single.instance.TransportI\
  1669. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1670. nsportInstanceSingleOperationAction.java:239)
  1671.  
  1672. at org.elasticsearch.transport.TransportRequestHandler.message\
  1673. Received(TransportRequestHandler.java:30)
  1674.  
  1675. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1676. essageReceived(RequestHandlerRegistry.java:66)
  1677.  
  1678. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1679. un(TcpTransport.java:1555)
  1680.  
  1681. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1682. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1683.  
  1684. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1685. un(AbstractRunnable.java:37)
  1686.  
  1687. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1688. lExecutor.java:1149)
  1689.  
  1690. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1691. olExecutor.java:624)
  1692.  
  1693. at java.lang.Thread.run(Thread.java:748)
  1694.  
  1695.  
  1696.  
  1697. "elasticsearch[fmfe16][index][T#17]" "
  1698. #428 "
  1699. daemon "
  1700. prio=5 "
  1701. os_prio=15 "
  1702. tid=0x0000000b02651800 "
  1703. nid=0x187c8 "
  1704. waiting on condition "
  1705. [0x00007fffd5856000]
  1706.  
  1707. java.lang.Thread.State: WAITING (parking)
  1708.  
  1709. at sun.misc.Unsafe.park(Native Method)
  1710.  
  1711. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1712. rrent.locks.ReentrantLock$NonfairSync)
  1713.  
  1714. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1715. a:175)
  1716.  
  1717. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1718. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1719.  
  1720. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1721. reQueued(AbstractQueuedSynchronizer.java:870)
  1722.  
  1723. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1724. re(AbstractQueuedSynchronizer.java:1199)
  1725.  
  1726. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1727. eentrantLock.java:209)
  1728.  
  1729. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1730. .java:285)
  1731.  
  1732. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1733. KeyedLock.java:76)
  1734.  
  1735. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1736. iveVersionMap.java:431)
  1737.  
  1738. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1739. ombstones(InternalEngine.java:1661)
  1740.  
  1741. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1742. etedTombstones(InternalEngine.java:1348)
  1743.  
  1744. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1745. nalEngine.java:1427)
  1746.  
  1747. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1748. ngine.java:637)
  1749.  
  1750. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1751. a:850)
  1752.  
  1753. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1754. etService.java:153)
  1755.  
  1756. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1757. vice.java:81)
  1758.  
  1759. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1760. Helper.java:74)
  1761.  
  1762. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1763. Operation(TransportUpdateAction.java:174)
  1764.  
  1765. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1766. Operation(TransportUpdateAction.java:167)
  1767.  
  1768. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1769. Operation(TransportUpdateAction.java:67)
  1770.  
  1771. at org.elasticsearch.action.support.single.instance.TransportI\
  1772. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1773. nsportInstanceSingleOperationAction.java:243)
  1774.  
  1775. at org.elasticsearch.action.support.single.instance.TransportI\
  1776. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1777. nsportInstanceSingleOperationAction.java:239)
  1778.  
  1779. at org.elasticsearch.transport.TransportRequestHandler.message\
  1780. Received(TransportRequestHandler.java:30)
  1781.  
  1782. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1783. essageReceived(RequestHandlerRegistry.java:66)
  1784.  
  1785. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1786. un(TcpTransport.java:1555)
  1787.  
  1788. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1789. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1790.  
  1791. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1792. un(AbstractRunnable.java:37)
  1793.  
  1794. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1795. lExecutor.java:1149)
  1796.  
  1797. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1798. olExecutor.java:624)
  1799.  
  1800. at java.lang.Thread.run(Thread.java:748)
  1801.  
  1802.  
  1803.  
  1804. "elasticsearch[fmfe16][index][T#16]" "
  1805. #427 "
  1806. daemon "
  1807. prio=5 "
  1808. os_prio=15 "
  1809. tid=0x0000000afc265800 "
  1810. nid=0x19cb5 "
  1811. waiting on condition "
  1812. [0x00007fffd6c6a000]
  1813.  
  1814. java.lang.Thread.State: WAITING (parking)
  1815.  
  1816. at sun.misc.Unsafe.park(Native Method)
  1817.  
  1818. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1819. rrent.locks.ReentrantLock$NonfairSync)
  1820.  
  1821. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1822. a:175)
  1823.  
  1824. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1825. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1826.  
  1827. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1828. reQueued(AbstractQueuedSynchronizer.java:870)
  1829.  
  1830. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1831. re(AbstractQueuedSynchronizer.java:1199)
  1832.  
  1833. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1834. eentrantLock.java:209)
  1835.  
  1836. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1837. .java:285)
  1838.  
  1839. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1840. KeyedLock.java:76)
  1841.  
  1842. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1843. iveVersionMap.java:431)
  1844.  
  1845. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1846. ombstones(InternalEngine.java:1661)
  1847.  
  1848. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1849. etedTombstones(InternalEngine.java:1348)
  1850.  
  1851. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1852. nalEngine.java:1427)
  1853.  
  1854. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1855. ngine.java:637)
  1856.  
  1857. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1858. a:850)
  1859.  
  1860. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1861. etService.java:153)
  1862.  
  1863. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1864. vice.java:81)
  1865.  
  1866. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1867. Helper.java:74)
  1868.  
  1869. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1870. Operation(TransportUpdateAction.java:174)
  1871.  
  1872. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1873. Operation(TransportUpdateAction.java:167)
  1874.  
  1875. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1876. Operation(TransportUpdateAction.java:67)
  1877.  
  1878. at org.elasticsearch.action.support.single.instance.TransportI\
  1879. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1880. nsportInstanceSingleOperationAction.java:243)
  1881.  
  1882. at org.elasticsearch.action.support.single.instance.TransportI\
  1883. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1884. nsportInstanceSingleOperationAction.java:239)
  1885.  
  1886. at org.elasticsearch.transport.TransportRequestHandler.message\
  1887. Received(TransportRequestHandler.java:30)
  1888.  
  1889. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1890. essageReceived(RequestHandlerRegistry.java:66)
  1891.  
  1892. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  1893. un(TcpTransport.java:1555)
  1894.  
  1895. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  1896. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  1897.  
  1898. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  1899. un(AbstractRunnable.java:37)
  1900.  
  1901. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  1902. lExecutor.java:1149)
  1903.  
  1904. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  1905. olExecutor.java:624)
  1906.  
  1907. at java.lang.Thread.run(Thread.java:748)
  1908.  
  1909.  
  1910.  
  1911. "elasticsearch[fmfe16][index][T#15]" "
  1912. #426 "
  1913. daemon "
  1914. prio=5 "
  1915. os_prio=15 "
  1916. tid=0x0000000ae97fe800 "
  1917. nid=0x19cb2 "
  1918. waiting on condition "
  1919. [0x00007fffd6a68000]
  1920.  
  1921. java.lang.Thread.State: WAITING (parking)
  1922.  
  1923. at sun.misc.Unsafe.park(Native Method)
  1924.  
  1925. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  1926. rrent.locks.ReentrantLock$NonfairSync)
  1927.  
  1928. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  1929. a:175)
  1930.  
  1931. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  1932. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  1933.  
  1934. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1935. reQueued(AbstractQueuedSynchronizer.java:870)
  1936.  
  1937. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  1938. re(AbstractQueuedSynchronizer.java:1199)
  1939.  
  1940. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  1941. eentrantLock.java:209)
  1942.  
  1943. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  1944. .java:285)
  1945.  
  1946. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  1947. KeyedLock.java:76)
  1948.  
  1949. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  1950. iveVersionMap.java:431)
  1951.  
  1952. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  1953. ombstones(InternalEngine.java:1661)
  1954.  
  1955. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  1956. etedTombstones(InternalEngine.java:1348)
  1957.  
  1958. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  1959. nalEngine.java:1427)
  1960.  
  1961. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  1962. ngine.java:637)
  1963.  
  1964. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  1965. a:850)
  1966.  
  1967. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  1968. etService.java:153)
  1969.  
  1970. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  1971. vice.java:81)
  1972.  
  1973. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  1974. Helper.java:74)
  1975.  
  1976. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1977. Operation(TransportUpdateAction.java:174)
  1978.  
  1979. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1980. Operation(TransportUpdateAction.java:167)
  1981.  
  1982. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  1983. Operation(TransportUpdateAction.java:67)
  1984.  
  1985. at org.elasticsearch.action.support.single.instance.TransportI\
  1986. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1987. nsportInstanceSingleOperationAction.java:243)
  1988.  
  1989. at org.elasticsearch.action.support.single.instance.TransportI\
  1990. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  1991. nsportInstanceSingleOperationAction.java:239)
  1992.  
  1993. at org.elasticsearch.transport.TransportRequestHandler.message\
  1994. Received(TransportRequestHandler.java:30)
  1995.  
  1996. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  1997. essageReceived(RequestHandlerRegistry.java:66)
  1998.  
  1999. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  2000. un(TcpTransport.java:1555)
  2001.  
  2002. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  2003. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  2004.  
  2005. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2006. un(AbstractRunnable.java:37)
  2007.  
  2008. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  2009. lExecutor.java:1149)
  2010.  
  2011. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  2012. olExecutor.java:624)
  2013.  
  2014. at java.lang.Thread.run(Thread.java:748)
  2015.  
  2016.  
  2017.  
  2018. "elasticsearch[fmfe16][index][T#14]" "
  2019. #425 "
  2020. daemon "
  2021. prio=5 "
  2022. os_prio=15 "
  2023. tid=0x0000000b0122e000 "
  2024. nid=0x19cb1 "
  2025. waiting on condition "
  2026. [0x00007fffc8f8e000]
  2027.  
  2028. java.lang.Thread.State: WAITING (parking)
  2029.  
  2030. at sun.misc.Unsafe.park(Native Method)
  2031.  
  2032. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  2033. rrent.locks.ReentrantLock$NonfairSync)
  2034.  
  2035. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  2036. a:175)
  2037.  
  2038. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  2039. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2040.  
  2041. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2042. reQueued(AbstractQueuedSynchronizer.java:870)
  2043.  
  2044. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2045. re(AbstractQueuedSynchronizer.java:1199)
  2046.  
  2047. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  2048. eentrantLock.java:209)
  2049.  
  2050. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  2051. .java:285)
  2052.  
  2053. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  2054. KeyedLock.java:76)
  2055.  
  2056. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  2057. iveVersionMap.java:431)
  2058.  
  2059. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  2060. ombstones(InternalEngine.java:1661)
  2061.  
  2062. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  2063. etedTombstones(InternalEngine.java:1348)
  2064.  
  2065. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  2066. nalEngine.java:1427)
  2067.  
  2068. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  2069. ngine.java:637)
  2070.  
  2071. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  2072. a:850)
  2073.  
  2074. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  2075. etService.java:153)
  2076.  
  2077. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  2078. vice.java:81)
  2079.  
  2080. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  2081. Helper.java:74)
  2082.  
  2083. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2084. Operation(TransportUpdateAction.java:174)
  2085.  
  2086. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2087. Operation(TransportUpdateAction.java:167)
  2088.  
  2089. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2090. Operation(TransportUpdateAction.java:67)
  2091.  
  2092. at org.elasticsearch.action.support.single.instance.TransportI\
  2093. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  2094. nsportInstanceSingleOperationAction.java:243)
  2095.  
  2096. at org.elasticsearch.action.support.single.instance.TransportI\
  2097. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  2098. nsportInstanceSingleOperationAction.java:239)
  2099.  
  2100. at org.elasticsearch.transport.TransportRequestHandler.message\
  2101. Received(TransportRequestHandler.java:30)
  2102.  
  2103. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  2104. essageReceived(RequestHandlerRegistry.java:66)
  2105.  
  2106. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  2107. un(TcpTransport.java:1555)
  2108.  
  2109. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  2110. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  2111.  
  2112. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2113. un(AbstractRunnable.java:37)
  2114.  
  2115. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  2116. lExecutor.java:1149)
  2117.  
  2118. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  2119. olExecutor.java:624)
  2120.  
  2121. at java.lang.Thread.run(Thread.java:748)
  2122.  
  2123.  
  2124.  
  2125. "elasticsearch[fmfe16][index][T#13]" "
  2126. #424 "
  2127. daemon "
  2128. prio=5 "
  2129. os_prio=15 "
  2130. tid=0x0000000af8d37000 "
  2131. nid=0x19c9a "
  2132. waiting on condition "
  2133. [0x00007fffc8887000]
  2134.  
  2135. java.lang.Thread.State: WAITING (parking)
  2136.  
  2137. at sun.misc.Unsafe.park(Native Method)
  2138.  
  2139. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  2140. rrent.locks.ReentrantLock$NonfairSync)
  2141.  
  2142. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  2143. a:175)
  2144.  
  2145. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  2146. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2147.  
  2148. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2149. reQueued(AbstractQueuedSynchronizer.java:870)
  2150.  
  2151. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2152. re(AbstractQueuedSynchronizer.java:1199)
  2153.  
  2154. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  2155. eentrantLock.java:209)
  2156.  
  2157. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  2158. .java:285)
  2159.  
  2160. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  2161. KeyedLock.java:76)
  2162.  
  2163. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  2164. iveVersionMap.java:431)
  2165.  
  2166. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  2167. ombstones(InternalEngine.java:1661)
  2168.  
  2169. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  2170. etedTombstones(InternalEngine.java:1348)
  2171.  
  2172. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  2173. nalEngine.java:1427)
  2174.  
  2175. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  2176. ngine.java:637)
  2177.  
  2178. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  2179. a:850)
  2180.  
  2181. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  2182. etService.java:153)
  2183.  
  2184. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  2185. vice.java:81)
  2186.  
  2187. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  2188. Helper.java:74)
  2189.  
  2190. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2191. Operation(TransportUpdateAction.java:174)
  2192.  
  2193. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2194. Operation(TransportUpdateAction.java:167)
  2195.  
  2196. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2197. Operation(TransportUpdateAction.java:67)
  2198.  
  2199. at org.elasticsearch.action.support.single.instance.TransportI\
  2200. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  2201. nsportInstanceSingleOperationAction.java:243)
  2202.  
  2203. at org.elasticsearch.action.support.single.instance.TransportI\
  2204. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  2205. nsportInstanceSingleOperationAction.java:239)
  2206.  
  2207. at org.elasticsearch.transport.TransportRequestHandler.message\
  2208. Received(TransportRequestHandler.java:30)
  2209.  
  2210. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  2211. essageReceived(RequestHandlerRegistry.java:66)
  2212.  
  2213. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  2214. un(TcpTransport.java:1555)
  2215.  
  2216. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  2217. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  2218.  
  2219. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2220. un(AbstractRunnable.java:37)
  2221.  
  2222. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  2223. lExecutor.java:1149)
  2224.  
  2225. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  2226. olExecutor.java:624)
  2227.  
  2228. at java.lang.Thread.run(Thread.java:748)
  2229.  
  2230.  
  2231.  
  2232. "elasticsearch[fmfe16][index][T#12]" "
  2233. #423 "
  2234. daemon "
  2235. prio=5 "
  2236. os_prio=15 "
  2237. tid=0x0000000afde8b800 "
  2238. nid=0x18cfb "
  2239. waiting on condition "
  2240. [0x00007fffd5b59000]
  2241.  
  2242. java.lang.Thread.State: WAITING (parking)
  2243.  
  2244. at sun.misc.Unsafe.park(Native Method)
  2245.  
  2246. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  2247. rrent.locks.ReentrantLock$NonfairSync)
  2248.  
  2249. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  2250. a:175)
  2251.  
  2252. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  2253. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2254.  
  2255. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2256. reQueued(AbstractQueuedSynchronizer.java:870)
  2257.  
  2258. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2259. re(AbstractQueuedSynchronizer.java:1199)
  2260.  
  2261. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  2262. eentrantLock.java:209)
  2263.  
  2264. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  2265. .java:285)
  2266.  
  2267. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  2268. KeyedLock.java:76)
  2269.  
  2270. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  2271. iveVersionMap.java:431)
  2272.  
  2273. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  2274. ombstones(InternalEngine.java:1661)
  2275.  
  2276. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  2277. etedTombstones(InternalEngine.java:1348)
  2278.  
  2279. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  2280. nalEngine.java:1427)
  2281.  
  2282. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  2283. ngine.java:637)
  2284.  
  2285. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  2286. a:850)
  2287.  
  2288. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  2289. etService.java:153)
  2290.  
  2291. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  2292. vice.java:81)
  2293.  
  2294. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  2295. Helper.java:74)
  2296.  
  2297. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2298. Operation(TransportUpdateAction.java:174)
  2299.  
  2300. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2301. Operation(TransportUpdateAction.java:167)
  2302.  
  2303. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  2304. Operation(TransportUpdateAction.java:67)
  2305.  
  2306. at org.elasticsearch.action.support.single.instance.TransportI\
  2307. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  2308. nsportInstanceSingleOperationAction.java:243)
  2309.  
  2310. at org.elasticsearch.action.support.single.instance.TransportI\
  2311. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  2312. nsportInstanceSingleOperationAction.java:239)
  2313.  
  2314. at org.elasticsearch.transport.TransportRequestHandler.message\
  2315. Received(TransportRequestHandler.java:30)
  2316.  
  2317. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  2318. essageReceived(RequestHandlerRegistry.java:66)
  2319.  
  2320. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  2321. un(TcpTransport.java:1555)
  2322.  
  2323. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  2324. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  2325.  
  2326. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2327. un(AbstractRunnable.java:37)
  2328.  
  2329. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  2330. lExecutor.java:1149)
  2331.  
  2332. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  2333. olExecutor.java:624)
  2334.  
  2335. at java.lang.Thread.run(Thread.java:748)
  2336.  
  2337.  
  2338.  
  2339. "elasticsearch[fmfe16][bulk][T#24]" "
  2340. #422 "
  2341. daemon "
  2342. prio=5 "
  2343. os_prio=15 "
  2344. tid=0x0000000b01081000 "
  2345. nid=0x18cf2 "
  2346. waiting on condition "
  2347. [0x00007fffd5755000]
  2348.  
  2349. java.lang.Thread.State: WAITING (parking)
  2350.  
  2351. at sun.misc.Unsafe.park(Native Method)
  2352.  
  2353. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  2354. rrent.locks.ReentrantLock$NonfairSync)
  2355.  
  2356. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  2357. a:175)
  2358.  
  2359. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  2360. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2361.  
  2362. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2363. reQueued(AbstractQueuedSynchronizer.java:870)
  2364.  
  2365. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2366. re(AbstractQueuedSynchronizer.java:1199)
  2367.  
  2368. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  2369. eentrantLock.java:209)
  2370.  
  2371. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  2372. .java:285)
  2373.  
  2374. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  2375. KeyedLock.java:76)
  2376.  
  2377. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  2378. iveVersionMap.java:431)
  2379.  
  2380. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  2381. ombstones(InternalEngine.java:1661)
  2382.  
  2383. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  2384. etedTombstones(InternalEngine.java:1348)
  2385.  
  2386. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  2387. alEngine.java:1211)
  2388.  
  2389. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  2390. java:839)
  2391.  
  2392. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  2393. on(IndexShard.java:806)
  2394.  
  2395. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  2396. onOnPrimary(IndexShard.java:770)
  2397.  
  2398. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2399. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  2400.  
  2401. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2402. uteDeleteRequest(TransportShardBulkAction.java:154)
  2403.  
  2404. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2405. uteBulkItemRequest(TransportShardBulkAction.java:244)
  2406.  
  2407. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  2408. ormOnPrimary(TransportShardBulkAction.java:123)
  2409.  
  2410. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  2411. dOperationOnPrimary(TransportShardBulkAction.java:110)
  2412.  
  2413. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  2414. dOperationOnPrimary(TransportShardBulkAction.java:72)
  2415.  
  2416. at org.elasticsearch.action.support.replication.TransportRepli\
  2417. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  2418. java:1034)
  2419.  
  2420. at org.elasticsearch.action.support.replication.TransportRepli\
  2421. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  2422. java:1012)
  2423.  
  2424. at org.elasticsearch.action.support.replication.ReplicationOpe\
  2425. ration.execute(ReplicationOperation.java:103)
  2426.  
  2427. at org.elasticsearch.action.support.replication.TransportRepli\
  2428. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  2429. java:359)
  2430.  
  2431. at org.elasticsearch.action.support.replication.TransportRepli\
  2432. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  2433. java:299)
  2434.  
  2435. at org.elasticsearch.action.support.replication.TransportRepli\
  2436. cationAction$1.onResponse(TransportReplicationAction.java:975)
  2437.  
  2438. at org.elasticsearch.action.support.replication.TransportRepli\
  2439. cationAction$1.onResponse(TransportReplicationAction.java:972)
  2440.  
  2441. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  2442. quire(IndexShardOperationPermits.java:238)
  2443.  
  2444. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  2445. ationPermit(IndexShard.java:2220)
  2446.  
  2447. at org.elasticsearch.action.support.replication.TransportRepli\
  2448. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  2449. ava:984)
  2450.  
  2451. at org.elasticsearch.action.support.replication.TransportRepli\
  2452. cationAction.access$500(TransportReplicationAction.java:98)
  2453.  
  2454. at org.elasticsearch.action.support.replication.TransportRepli\
  2455. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  2456. 320)
  2457.  
  2458. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2459. un(AbstractRunnable.java:37)
  2460.  
  2461. at org.elasticsearch.action.support.replication.TransportRepli\
  2462. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  2463. tReplicationAction.java:295)
  2464.  
  2465. at org.elasticsearch.action.support.replication.TransportRepli\
  2466. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  2467. tReplicationAction.java:282)
  2468.  
  2469. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  2470. essageReceived(RequestHandlerRegistry.java:66)
  2471.  
  2472. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  2473. ortService.java:656)
  2474.  
  2475. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  2476. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  2477.  
  2478. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2479. un(AbstractRunnable.java:37)
  2480.  
  2481. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  2482. lExecutor.java:1149)
  2483.  
  2484. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  2485. olExecutor.java:624)
  2486.  
  2487. at java.lang.Thread.run(Thread.java:748)
  2488.  
  2489.  
  2490.  
  2491. "elasticsearch[fmfe16][bulk][T#23]" "
  2492. #421 "
  2493. daemon "
  2494. prio=5 "
  2495. os_prio=15 "
  2496. tid=0x0000000b00aa7800 "
  2497. nid=0x187a2 "
  2498. waiting on condition "
  2499. [0x00007fffca5a4000]
  2500.  
  2501. java.lang.Thread.State: WAITING (parking)
  2502.  
  2503. at sun.misc.Unsafe.park(Native Method)
  2504.  
  2505. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  2506. rrent.locks.ReentrantLock$NonfairSync)
  2507.  
  2508. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  2509. a:175)
  2510.  
  2511. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  2512. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2513.  
  2514. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2515. reQueued(AbstractQueuedSynchronizer.java:870)
  2516.  
  2517. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2518. re(AbstractQueuedSynchronizer.java:1199)
  2519.  
  2520. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  2521. eentrantLock.java:209)
  2522.  
  2523. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  2524. .java:285)
  2525.  
  2526. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  2527. KeyedLock.java:76)
  2528.  
  2529. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  2530. iveVersionMap.java:431)
  2531.  
  2532. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  2533. ombstones(InternalEngine.java:1661)
  2534.  
  2535. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  2536. etedTombstones(InternalEngine.java:1348)
  2537.  
  2538. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  2539. alEngine.java:1211)
  2540.  
  2541. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  2542. java:839)
  2543.  
  2544. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  2545. on(IndexShard.java:806)
  2546.  
  2547. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  2548. onOnPrimary(IndexShard.java:770)
  2549.  
  2550. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2551. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  2552.  
  2553. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2554. uteDeleteRequest(TransportShardBulkAction.java:154)
  2555.  
  2556. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2557. uteBulkItemRequest(TransportShardBulkAction.java:244)
  2558.  
  2559. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  2560. ormOnPrimary(TransportShardBulkAction.java:123)
  2561.  
  2562. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  2563. dOperationOnPrimary(TransportShardBulkAction.java:110)
  2564.  
  2565. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  2566. dOperationOnPrimary(TransportShardBulkAction.java:72)
  2567.  
  2568. at org.elasticsearch.action.support.replication.TransportRepli\
  2569. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  2570. java:1034)
  2571.  
  2572. at org.elasticsearch.action.support.replication.TransportRepli\
  2573. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  2574. java:1012)
  2575.  
  2576. at org.elasticsearch.action.support.replication.ReplicationOpe\
  2577. ration.execute(ReplicationOperation.java:103)
  2578.  
  2579. at org.elasticsearch.action.support.replication.TransportRepli\
  2580. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  2581. java:359)
  2582.  
  2583. at org.elasticsearch.action.support.replication.TransportRepli\
  2584. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  2585. java:299)
  2586.  
  2587. at org.elasticsearch.action.support.replication.TransportRepli\
  2588. cationAction$1.onResponse(TransportReplicationAction.java:975)
  2589.  
  2590. at org.elasticsearch.action.support.replication.TransportRepli\
  2591. cationAction$1.onResponse(TransportReplicationAction.java:972)
  2592.  
  2593. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  2594. quire(IndexShardOperationPermits.java:238)
  2595.  
  2596. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  2597. ationPermit(IndexShard.java:2220)
  2598.  
  2599. at org.elasticsearch.action.support.replication.TransportRepli\
  2600. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  2601. ava:984)
  2602.  
  2603. at org.elasticsearch.action.support.replication.TransportRepli\
  2604. cationAction.access$500(TransportReplicationAction.java:98)
  2605.  
  2606. at org.elasticsearch.action.support.replication.TransportRepli\
  2607. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  2608. 320)
  2609.  
  2610. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2611. un(AbstractRunnable.java:37)
  2612.  
  2613. at org.elasticsearch.action.support.replication.TransportRepli\
  2614. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  2615. tReplicationAction.java:295)
  2616.  
  2617. at org.elasticsearch.action.support.replication.TransportRepli\
  2618. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  2619. tReplicationAction.java:282)
  2620.  
  2621. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  2622. essageReceived(RequestHandlerRegistry.java:66)
  2623.  
  2624. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  2625. ortService.java:656)
  2626.  
  2627. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  2628. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  2629.  
  2630. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2631. un(AbstractRunnable.java:37)
  2632.  
  2633. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  2634. lExecutor.java:1149)
  2635.  
  2636. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  2637. olExecutor.java:624)
  2638.  
  2639. at java.lang.Thread.run(Thread.java:748)
  2640.  
  2641.  
  2642.  
  2643. "elasticsearch[fmfe16][bulk][T#22]" "
  2644. #418 "
  2645. daemon "
  2646. prio=5 "
  2647. os_prio=15 "
  2648. tid=0x0000000b0122d000 "
  2649. nid=0x19789 "
  2650. waiting on condition "
  2651. [0x00007fffc9695000]
  2652.  
  2653. java.lang.Thread.State: WAITING (parking)
  2654.  
  2655. at sun.misc.Unsafe.park(Native Method)
  2656.  
  2657. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  2658. rrent.locks.ReentrantLock$NonfairSync)
  2659.  
  2660. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  2661. a:175)
  2662.  
  2663. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  2664. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2665.  
  2666. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2667. reQueued(AbstractQueuedSynchronizer.java:870)
  2668.  
  2669. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2670. re(AbstractQueuedSynchronizer.java:1199)
  2671.  
  2672. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  2673. eentrantLock.java:209)
  2674.  
  2675. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  2676. .java:285)
  2677.  
  2678. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  2679. KeyedLock.java:76)
  2680.  
  2681. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  2682. iveVersionMap.java:431)
  2683.  
  2684. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  2685. ombstones(InternalEngine.java:1661)
  2686.  
  2687. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  2688. etedTombstones(InternalEngine.java:1348)
  2689.  
  2690. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  2691. nalEngine.java:1427)
  2692.  
  2693. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  2694. ngine.java:637)
  2695.  
  2696. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  2697. a:850)
  2698.  
  2699. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  2700. etService.java:153)
  2701.  
  2702. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  2703. vice.java:81)
  2704.  
  2705. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  2706. Helper.java:74)
  2707.  
  2708. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2709. uteUpdateRequestOnce(TransportShardBulkAction.java:343)
  2710.  
  2711. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2712. uteUpdateRequest(TransportShardBulkAction.java:406)
  2713.  
  2714. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2715. uteBulkItemRequest(TransportShardBulkAction.java:240)
  2716.  
  2717. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  2718. ormOnPrimary(TransportShardBulkAction.java:123)
  2719.  
  2720. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  2721. dOperationOnPrimary(TransportShardBulkAction.java:110)
  2722.  
  2723. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  2724. dOperationOnPrimary(TransportShardBulkAction.java:72)
  2725.  
  2726. at org.elasticsearch.action.support.replication.TransportRepli\
  2727. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  2728. java:1034)
  2729.  
  2730. at org.elasticsearch.action.support.replication.TransportRepli\
  2731. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  2732. java:1012)
  2733.  
  2734. at org.elasticsearch.action.support.replication.ReplicationOpe\
  2735. ration.execute(ReplicationOperation.java:103)
  2736.  
  2737. at org.elasticsearch.action.support.replication.TransportRepli\
  2738. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  2739. java:359)
  2740.  
  2741. at org.elasticsearch.action.support.replication.TransportRepli\
  2742. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  2743. java:299)
  2744.  
  2745. at org.elasticsearch.action.support.replication.TransportRepli\
  2746. cationAction$1.onResponse(TransportReplicationAction.java:975)
  2747.  
  2748. at org.elasticsearch.action.support.replication.TransportRepli\
  2749. cationAction$1.onResponse(TransportReplicationAction.java:972)
  2750.  
  2751. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  2752. quire(IndexShardOperationPermits.java:238)
  2753.  
  2754. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  2755. ationPermit(IndexShard.java:2220)
  2756.  
  2757. at org.elasticsearch.action.support.replication.TransportRepli\
  2758. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  2759. ava:984)
  2760.  
  2761. at org.elasticsearch.action.support.replication.TransportRepli\
  2762. cationAction.access$500(TransportReplicationAction.java:98)
  2763.  
  2764. at org.elasticsearch.action.support.replication.TransportRepli\
  2765. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  2766. 320)
  2767.  
  2768. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2769. un(AbstractRunnable.java:37)
  2770.  
  2771. at org.elasticsearch.action.support.replication.TransportRepli\
  2772. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  2773. tReplicationAction.java:295)
  2774.  
  2775. at org.elasticsearch.action.support.replication.TransportRepli\
  2776. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  2777. tReplicationAction.java:282)
  2778.  
  2779. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  2780. essageReceived(RequestHandlerRegistry.java:66)
  2781.  
  2782. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  2783. ortService.java:656)
  2784.  
  2785. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  2786. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  2787.  
  2788. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2789. un(AbstractRunnable.java:37)
  2790.  
  2791. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  2792. lExecutor.java:1149)
  2793.  
  2794. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  2795. olExecutor.java:624)
  2796.  
  2797. at java.lang.Thread.run(Thread.java:748)
  2798.  
  2799.  
  2800.  
  2801. "elasticsearch[fmfe16][bulk][T#21]" "
  2802. #417 "
  2803. daemon "
  2804. prio=5 "
  2805. os_prio=15 "
  2806. tid=0x0000000aea3c5000 "
  2807. nid=0x194b5 "
  2808. waiting on condition "
  2809. [0x00007fffc9190000]
  2810.  
  2811. java.lang.Thread.State: WAITING (parking)
  2812.  
  2813. at sun.misc.Unsafe.park(Native Method)
  2814.  
  2815. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  2816. rrent.locks.ReentrantLock$NonfairSync)
  2817.  
  2818. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  2819. a:175)
  2820.  
  2821. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  2822. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2823.  
  2824. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2825. reQueued(AbstractQueuedSynchronizer.java:870)
  2826.  
  2827. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2828. re(AbstractQueuedSynchronizer.java:1199)
  2829.  
  2830. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  2831. eentrantLock.java:209)
  2832.  
  2833. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  2834. .java:285)
  2835.  
  2836. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  2837. KeyedLock.java:76)
  2838.  
  2839. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  2840. iveVersionMap.java:431)
  2841.  
  2842. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  2843. ombstones(InternalEngine.java:1661)
  2844.  
  2845. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  2846. etedTombstones(InternalEngine.java:1348)
  2847.  
  2848. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  2849. alEngine.java:1211)
  2850.  
  2851. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  2852. java:839)
  2853.  
  2854. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  2855. on(IndexShard.java:806)
  2856.  
  2857. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  2858. onOnPrimary(IndexShard.java:770)
  2859.  
  2860. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2861. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  2862.  
  2863. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2864. uteDeleteRequest(TransportShardBulkAction.java:154)
  2865.  
  2866. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  2867. uteBulkItemRequest(TransportShardBulkAction.java:244)
  2868.  
  2869. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  2870. ormOnPrimary(TransportShardBulkAction.java:123)
  2871.  
  2872. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  2873. dOperationOnPrimary(TransportShardBulkAction.java:110)
  2874.  
  2875. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  2876. dOperationOnPrimary(TransportShardBulkAction.java:72)
  2877.  
  2878. at org.elasticsearch.action.support.replication.TransportRepli\
  2879. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  2880. java:1034)
  2881.  
  2882. at org.elasticsearch.action.support.replication.TransportRepli\
  2883. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  2884. java:1012)
  2885.  
  2886. at org.elasticsearch.action.support.replication.ReplicationOpe\
  2887. ration.execute(ReplicationOperation.java:103)
  2888.  
  2889. at org.elasticsearch.action.support.replication.TransportRepli\
  2890. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  2891. java:359)
  2892.  
  2893. at org.elasticsearch.action.support.replication.TransportRepli\
  2894. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  2895. java:299)
  2896.  
  2897. at org.elasticsearch.action.support.replication.TransportRepli\
  2898. cationAction$1.onResponse(TransportReplicationAction.java:975)
  2899.  
  2900. at org.elasticsearch.action.support.replication.TransportRepli\
  2901. cationAction$1.onResponse(TransportReplicationAction.java:972)
  2902.  
  2903. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  2904. quire(IndexShardOperationPermits.java:238)
  2905.  
  2906. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  2907. ationPermit(IndexShard.java:2220)
  2908.  
  2909. at org.elasticsearch.action.support.replication.TransportRepli\
  2910. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  2911. ava:984)
  2912.  
  2913. at org.elasticsearch.action.support.replication.TransportRepli\
  2914. cationAction.access$500(TransportReplicationAction.java:98)
  2915.  
  2916. at org.elasticsearch.action.support.replication.TransportRepli\
  2917. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  2918. 320)
  2919.  
  2920. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2921. un(AbstractRunnable.java:37)
  2922.  
  2923. at org.elasticsearch.action.support.replication.TransportRepli\
  2924. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  2925. tReplicationAction.java:295)
  2926.  
  2927. at org.elasticsearch.action.support.replication.TransportRepli\
  2928. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  2929. tReplicationAction.java:282)
  2930.  
  2931. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  2932. essageReceived(RequestHandlerRegistry.java:66)
  2933.  
  2934. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  2935. ortService.java:656)
  2936.  
  2937. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  2938. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  2939.  
  2940. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  2941. un(AbstractRunnable.java:37)
  2942.  
  2943. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  2944. lExecutor.java:1149)
  2945.  
  2946. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  2947. olExecutor.java:624)
  2948.  
  2949. at java.lang.Thread.run(Thread.java:748)
  2950.  
  2951.  
  2952.  
  2953. "elasticsearch[fmfe16][index][T#11]" "
  2954. #416 "
  2955. daemon "
  2956. prio=5 "
  2957. os_prio=15 "
  2958. tid=0x0000000b0c6b3000 "
  2959. nid=0x194a1 "
  2960. waiting on condition "
  2961. [0x00007fffd6967000]
  2962.  
  2963. java.lang.Thread.State: WAITING (parking)
  2964.  
  2965. at sun.misc.Unsafe.park(Native Method)
  2966.  
  2967. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  2968. rrent.locks.ReentrantLock$NonfairSync)
  2969.  
  2970. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  2971. a:175)
  2972.  
  2973. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  2974. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  2975.  
  2976. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2977. reQueued(AbstractQueuedSynchronizer.java:870)
  2978.  
  2979. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  2980. re(AbstractQueuedSynchronizer.java:1199)
  2981.  
  2982. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  2983. eentrantLock.java:209)
  2984.  
  2985. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  2986. .java:285)
  2987.  
  2988. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  2989. KeyedLock.java:76)
  2990.  
  2991. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  2992. iveVersionMap.java:431)
  2993.  
  2994. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  2995. ombstones(InternalEngine.java:1661)
  2996.  
  2997. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  2998. etedTombstones(InternalEngine.java:1348)
  2999.  
  3000. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  3001. nalEngine.java:1427)
  3002.  
  3003. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  3004. ngine.java:637)
  3005.  
  3006. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  3007. a:850)
  3008.  
  3009. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  3010. etService.java:153)
  3011.  
  3012. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  3013. vice.java:81)
  3014.  
  3015. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  3016. Helper.java:74)
  3017.  
  3018. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3019. Operation(TransportUpdateAction.java:174)
  3020.  
  3021. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3022. Operation(TransportUpdateAction.java:167)
  3023.  
  3024. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3025. Operation(TransportUpdateAction.java:67)
  3026.  
  3027. at org.elasticsearch.action.support.single.instance.TransportI\
  3028. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  3029. nsportInstanceSingleOperationAction.java:243)
  3030.  
  3031. at org.elasticsearch.action.support.single.instance.TransportI\
  3032. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  3033. nsportInstanceSingleOperationAction.java:239)
  3034.  
  3035. at org.elasticsearch.transport.TransportRequestHandler.message\
  3036. Received(TransportRequestHandler.java:30)
  3037.  
  3038. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  3039. essageReceived(RequestHandlerRegistry.java:66)
  3040.  
  3041. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  3042. un(TcpTransport.java:1555)
  3043.  
  3044. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  3045. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  3046.  
  3047. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3048. un(AbstractRunnable.java:37)
  3049.  
  3050. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  3051. lExecutor.java:1149)
  3052.  
  3053. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  3054. olExecutor.java:624)
  3055.  
  3056. at java.lang.Thread.run(Thread.java:748)
  3057.  
  3058.  
  3059.  
  3060. "elasticsearch[fmfe16][bulk][T#20]" "
  3061. #415 "
  3062. daemon "
  3063. prio=5 "
  3064. os_prio=15 "
  3065. tid=0x0000000b010f5800 "
  3066. nid=0x1949c "
  3067. waiting on condition "
  3068. [0x00007fffd6866000]
  3069.  
  3070. java.lang.Thread.State: WAITING (parking)
  3071.  
  3072. at sun.misc.Unsafe.park(Native Method)
  3073.  
  3074. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  3075. rrent.locks.ReentrantLock$NonfairSync)
  3076.  
  3077. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  3078. a:175)
  3079.  
  3080. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  3081. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  3082.  
  3083. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3084. reQueued(AbstractQueuedSynchronizer.java:870)
  3085.  
  3086. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3087. re(AbstractQueuedSynchronizer.java:1199)
  3088.  
  3089. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  3090. eentrantLock.java:209)
  3091.  
  3092. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  3093. .java:285)
  3094.  
  3095. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  3096. KeyedLock.java:76)
  3097.  
  3098. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  3099. iveVersionMap.java:431)
  3100.  
  3101. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  3102. ombstones(InternalEngine.java:1661)
  3103.  
  3104. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  3105. etedTombstones(InternalEngine.java:1348)
  3106.  
  3107. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  3108. alEngine.java:1211)
  3109.  
  3110. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  3111. java:839)
  3112.  
  3113. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  3114. on(IndexShard.java:806)
  3115.  
  3116. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  3117. onOnPrimary(IndexShard.java:770)
  3118.  
  3119. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3120. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  3121.  
  3122. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3123. uteDeleteRequest(TransportShardBulkAction.java:154)
  3124.  
  3125. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3126. uteBulkItemRequest(TransportShardBulkAction.java:244)
  3127.  
  3128. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  3129. ormOnPrimary(TransportShardBulkAction.java:123)
  3130.  
  3131. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  3132. dOperationOnPrimary(TransportShardBulkAction.java:110)
  3133.  
  3134. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  3135. dOperationOnPrimary(TransportShardBulkAction.java:72)
  3136.  
  3137. at org.elasticsearch.action.support.replication.TransportRepli\
  3138. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  3139. java:1034)
  3140.  
  3141. at org.elasticsearch.action.support.replication.TransportRepli\
  3142. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  3143. java:1012)
  3144.  
  3145. at org.elasticsearch.action.support.replication.ReplicationOpe\
  3146. ration.execute(ReplicationOperation.java:103)
  3147.  
  3148. at org.elasticsearch.action.support.replication.TransportRepli\
  3149. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  3150. java:359)
  3151.  
  3152. at org.elasticsearch.action.support.replication.TransportRepli\
  3153. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  3154. java:299)
  3155.  
  3156. at org.elasticsearch.action.support.replication.TransportRepli\
  3157. cationAction$1.onResponse(TransportReplicationAction.java:975)
  3158.  
  3159. at org.elasticsearch.action.support.replication.TransportRepli\
  3160. cationAction$1.onResponse(TransportReplicationAction.java:972)
  3161.  
  3162. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  3163. quire(IndexShardOperationPermits.java:238)
  3164.  
  3165. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  3166. ationPermit(IndexShard.java:2220)
  3167.  
  3168. at org.elasticsearch.action.support.replication.TransportRepli\
  3169. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  3170. ava:984)
  3171.  
  3172. at org.elasticsearch.action.support.replication.TransportRepli\
  3173. cationAction.access$500(TransportReplicationAction.java:98)
  3174.  
  3175. at org.elasticsearch.action.support.replication.TransportRepli\
  3176. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  3177. 320)
  3178.  
  3179. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3180. un(AbstractRunnable.java:37)
  3181.  
  3182. at org.elasticsearch.action.support.replication.TransportRepli\
  3183. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  3184. tReplicationAction.java:295)
  3185.  
  3186. at org.elasticsearch.action.support.replication.TransportRepli\
  3187. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  3188. tReplicationAction.java:282)
  3189.  
  3190. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  3191. essageReceived(RequestHandlerRegistry.java:66)
  3192.  
  3193. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  3194. ortService.java:656)
  3195.  
  3196. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  3197. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  3198.  
  3199. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3200. un(AbstractRunnable.java:37)
  3201.  
  3202. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  3203. lExecutor.java:1149)
  3204.  
  3205. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  3206. olExecutor.java:624)
  3207.  
  3208. at java.lang.Thread.run(Thread.java:748)
  3209.  
  3210.  
  3211.  
  3212. "elasticsearch[fmfe16][bulk][T#19]" "
  3213. #414 "
  3214. daemon "
  3215. prio=5 "
  3216. os_prio=15 "
  3217. tid=0x0000000ae9d5b000 "
  3218. nid=0x1949a "
  3219. waiting on condition "
  3220. [0x00007fffc908f000]
  3221.  
  3222. java.lang.Thread.State: WAITING (parking)
  3223.  
  3224. at sun.misc.Unsafe.park(Native Method)
  3225.  
  3226. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  3227. rrent.locks.ReentrantLock$NonfairSync)
  3228.  
  3229. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  3230. a:175)
  3231.  
  3232. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  3233. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  3234.  
  3235. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3236. reQueued(AbstractQueuedSynchronizer.java:870)
  3237.  
  3238. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3239. re(AbstractQueuedSynchronizer.java:1199)
  3240.  
  3241. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  3242. eentrantLock.java:209)
  3243.  
  3244. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  3245. .java:285)
  3246.  
  3247. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  3248. KeyedLock.java:76)
  3249.  
  3250. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  3251. iveVersionMap.java:431)
  3252.  
  3253. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  3254. ombstones(InternalEngine.java:1661)
  3255.  
  3256. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  3257. etedTombstones(InternalEngine.java:1348)
  3258.  
  3259. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  3260. alEngine.java:1211)
  3261.  
  3262. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  3263. java:839)
  3264.  
  3265. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  3266. on(IndexShard.java:806)
  3267.  
  3268. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  3269. onOnPrimary(IndexShard.java:770)
  3270.  
  3271. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3272. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  3273.  
  3274. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3275. uteDeleteRequest(TransportShardBulkAction.java:154)
  3276.  
  3277. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3278. uteBulkItemRequest(TransportShardBulkAction.java:244)
  3279.  
  3280. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  3281. ormOnPrimary(TransportShardBulkAction.java:123)
  3282.  
  3283. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  3284. dOperationOnPrimary(TransportShardBulkAction.java:110)
  3285.  
  3286. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  3287. dOperationOnPrimary(TransportShardBulkAction.java:72)
  3288.  
  3289. at org.elasticsearch.action.support.replication.TransportRepli\
  3290. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  3291. java:1034)
  3292.  
  3293. at org.elasticsearch.action.support.replication.TransportRepli\
  3294. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  3295. java:1012)
  3296.  
  3297. at org.elasticsearch.action.support.replication.ReplicationOpe\
  3298. ration.execute(ReplicationOperation.java:103)
  3299.  
  3300. at org.elasticsearch.action.support.replication.TransportRepli\
  3301. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  3302. java:359)
  3303.  
  3304. at org.elasticsearch.action.support.replication.TransportRepli\
  3305. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  3306. java:299)
  3307.  
  3308. at org.elasticsearch.action.support.replication.TransportRepli\
  3309. cationAction$1.onResponse(TransportReplicationAction.java:975)
  3310.  
  3311. at org.elasticsearch.action.support.replication.TransportRepli\
  3312. cationAction$1.onResponse(TransportReplicationAction.java:972)
  3313.  
  3314. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  3315. quire(IndexShardOperationPermits.java:238)
  3316.  
  3317. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  3318. ationPermit(IndexShard.java:2220)
  3319.  
  3320. at org.elasticsearch.action.support.replication.TransportRepli\
  3321. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  3322. ava:984)
  3323.  
  3324. at org.elasticsearch.action.support.replication.TransportRepli\
  3325. cationAction.access$500(TransportReplicationAction.java:98)
  3326.  
  3327. at org.elasticsearch.action.support.replication.TransportRepli\
  3328. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  3329. 320)
  3330.  
  3331. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3332. un(AbstractRunnable.java:37)
  3333.  
  3334. at org.elasticsearch.action.support.replication.TransportRepli\
  3335. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  3336. tReplicationAction.java:295)
  3337.  
  3338. at org.elasticsearch.action.support.replication.TransportRepli\
  3339. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  3340. tReplicationAction.java:282)
  3341.  
  3342. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  3343. essageReceived(RequestHandlerRegistry.java:66)
  3344.  
  3345. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  3346. ortService.java:656)
  3347.  
  3348. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  3349. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  3350.  
  3351. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3352. un(AbstractRunnable.java:37)
  3353.  
  3354. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  3355. lExecutor.java:1149)
  3356.  
  3357. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  3358. olExecutor.java:624)
  3359.  
  3360. at java.lang.Thread.run(Thread.java:748)
  3361.  
  3362.  
  3363.  
  3364. "elasticsearch[fmfe16][index][T#10]" "
  3365. #413 "
  3366. daemon "
  3367. prio=5 "
  3368. os_prio=15 "
  3369. tid=0x0000000b02650800 "
  3370. nid=0x1947f "
  3371. waiting on condition "
  3372. [0x00007fffca7a6000]
  3373.  
  3374. java.lang.Thread.State: WAITING (parking)
  3375.  
  3376. at sun.misc.Unsafe.park(Native Method)
  3377.  
  3378. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  3379. rrent.locks.ReentrantLock$NonfairSync)
  3380.  
  3381. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  3382. a:175)
  3383.  
  3384. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  3385. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  3386.  
  3387. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3388. reQueued(AbstractQueuedSynchronizer.java:870)
  3389.  
  3390. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3391. re(AbstractQueuedSynchronizer.java:1199)
  3392.  
  3393. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  3394. eentrantLock.java:209)
  3395.  
  3396. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  3397. .java:285)
  3398.  
  3399. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  3400. KeyedLock.java:76)
  3401.  
  3402. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  3403. iveVersionMap.java:431)
  3404.  
  3405. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  3406. ombstones(InternalEngine.java:1661)
  3407.  
  3408. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  3409. etedTombstones(InternalEngine.java:1348)
  3410.  
  3411. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  3412. nalEngine.java:1427)
  3413.  
  3414. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  3415. ngine.java:637)
  3416.  
  3417. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  3418. a:850)
  3419.  
  3420. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  3421. etService.java:153)
  3422.  
  3423. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  3424. vice.java:81)
  3425.  
  3426. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  3427. Helper.java:74)
  3428.  
  3429. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3430. Operation(TransportUpdateAction.java:174)
  3431.  
  3432. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3433. Operation(TransportUpdateAction.java:167)
  3434.  
  3435. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3436. Operation(TransportUpdateAction.java:67)
  3437.  
  3438. at org.elasticsearch.action.support.single.instance.TransportI\
  3439. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  3440. nsportInstanceSingleOperationAction.java:243)
  3441.  
  3442. at org.elasticsearch.action.support.single.instance.TransportI\
  3443. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  3444. nsportInstanceSingleOperationAction.java:239)
  3445.  
  3446. at org.elasticsearch.transport.TransportRequestHandler.message\
  3447. Received(TransportRequestHandler.java:30)
  3448.  
  3449. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  3450. essageReceived(RequestHandlerRegistry.java:66)
  3451.  
  3452. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  3453. un(TcpTransport.java:1555)
  3454.  
  3455. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  3456. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  3457.  
  3458. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3459. un(AbstractRunnable.java:37)
  3460.  
  3461. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  3462. lExecutor.java:1149)
  3463.  
  3464. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  3465. olExecutor.java:624)
  3466.  
  3467. at java.lang.Thread.run(Thread.java:748)
  3468.  
  3469.  
  3470.  
  3471. "elasticsearch[fmfe16][bulk][T#18]" "
  3472. #412 "
  3473. daemon "
  3474. prio=5 "
  3475. os_prio=15 "
  3476. tid=0x0000000b01330800 "
  3477. nid=0x1946c "
  3478. waiting on condition "
  3479. [0x00007fffd6260000]
  3480.  
  3481. java.lang.Thread.State: WAITING (parking)
  3482.  
  3483. at sun.misc.Unsafe.park(Native Method)
  3484.  
  3485. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  3486. rrent.locks.ReentrantLock$NonfairSync)
  3487.  
  3488. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  3489. a:175)
  3490.  
  3491. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  3492. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  3493.  
  3494. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3495. reQueued(AbstractQueuedSynchronizer.java:870)
  3496.  
  3497. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3498. re(AbstractQueuedSynchronizer.java:1199)
  3499.  
  3500. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  3501. eentrantLock.java:209)
  3502.  
  3503. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  3504. .java:285)
  3505.  
  3506. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  3507. KeyedLock.java:76)
  3508.  
  3509. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  3510. iveVersionMap.java:431)
  3511.  
  3512. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  3513. ombstones(InternalEngine.java:1661)
  3514.  
  3515. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  3516. etedTombstones(InternalEngine.java:1348)
  3517.  
  3518. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  3519. alEngine.java:1211)
  3520.  
  3521. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  3522. java:839)
  3523.  
  3524. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  3525. on(IndexShard.java:806)
  3526.  
  3527. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  3528. onOnPrimary(IndexShard.java:770)
  3529.  
  3530. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3531. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  3532.  
  3533. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3534. uteDeleteRequest(TransportShardBulkAction.java:154)
  3535.  
  3536. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3537. uteBulkItemRequest(TransportShardBulkAction.java:244)
  3538.  
  3539. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  3540. ormOnPrimary(TransportShardBulkAction.java:123)
  3541.  
  3542. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  3543. dOperationOnPrimary(TransportShardBulkAction.java:110)
  3544.  
  3545. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  3546. dOperationOnPrimary(TransportShardBulkAction.java:72)
  3547.  
  3548. at org.elasticsearch.action.support.replication.TransportRepli\
  3549. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  3550. java:1034)
  3551.  
  3552. at org.elasticsearch.action.support.replication.TransportRepli\
  3553. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  3554. java:1012)
  3555.  
  3556. at org.elasticsearch.action.support.replication.ReplicationOpe\
  3557. ration.execute(ReplicationOperation.java:103)
  3558.  
  3559. at org.elasticsearch.action.support.replication.TransportRepli\
  3560. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  3561. java:359)
  3562.  
  3563. at org.elasticsearch.action.support.replication.TransportRepli\
  3564. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  3565. java:299)
  3566.  
  3567. at org.elasticsearch.action.support.replication.TransportRepli\
  3568. cationAction$1.onResponse(TransportReplicationAction.java:975)
  3569.  
  3570. at org.elasticsearch.action.support.replication.TransportRepli\
  3571. cationAction$1.onResponse(TransportReplicationAction.java:972)
  3572.  
  3573. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  3574. quire(IndexShardOperationPermits.java:238)
  3575.  
  3576. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  3577. ationPermit(IndexShard.java:2220)
  3578.  
  3579. at org.elasticsearch.action.support.replication.TransportRepli\
  3580. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  3581. ava:984)
  3582.  
  3583. at org.elasticsearch.action.support.replication.TransportRepli\
  3584. cationAction.access$500(TransportReplicationAction.java:98)
  3585.  
  3586. at org.elasticsearch.action.support.replication.TransportRepli\
  3587. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  3588. 320)
  3589.  
  3590. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3591. un(AbstractRunnable.java:37)
  3592.  
  3593. at org.elasticsearch.action.support.replication.TransportRepli\
  3594. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  3595. tReplicationAction.java:295)
  3596.  
  3597. at org.elasticsearch.action.support.replication.TransportRepli\
  3598. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  3599. tReplicationAction.java:282)
  3600.  
  3601. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  3602. essageReceived(RequestHandlerRegistry.java:66)
  3603.  
  3604. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  3605. ortService.java:656)
  3606.  
  3607. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  3608. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  3609.  
  3610. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3611. un(AbstractRunnable.java:37)
  3612.  
  3613. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  3614. lExecutor.java:1149)
  3615.  
  3616. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  3617. olExecutor.java:624)
  3618.  
  3619. at java.lang.Thread.run(Thread.java:748)
  3620.  
  3621.  
  3622.  
  3623. "elasticsearch[fmfe16][index][T#9]" "
  3624. #411 "
  3625. daemon "
  3626. prio=5 "
  3627. os_prio=15 "
  3628. tid=0x0000000b03a38800 "
  3629. nid=0x19465 "
  3630. waiting for monitor entry "
  3631. [0x00007fffd615f000]
  3632.  
  3633. java.lang.Thread.State: BLOCKED (on object monitor)
  3634.  
  3635. at org.elasticsearch.index.engine.InternalEngine.getVersionFro\
  3636. mMap(InternalEngine.java:732)
  3637.  
  3638. - waiting to lock <0x00000008b40309c8> "
  3639. (a org.elasticsearch.index.engine.LiveVersionMap)
  3640.  
  3641. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  3642. ngine.java:627)
  3643.  
  3644. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  3645. a:850)
  3646.  
  3647. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  3648. etService.java:153)
  3649.  
  3650. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  3651. vice.java:81)
  3652.  
  3653. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  3654. Helper.java:74)
  3655.  
  3656. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3657. Operation(TransportUpdateAction.java:174)
  3658.  
  3659. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3660. Operation(TransportUpdateAction.java:167)
  3661.  
  3662. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3663. Operation(TransportUpdateAction.java:67)
  3664.  
  3665. at org.elasticsearch.action.support.single.instance.TransportI\
  3666. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  3667. nsportInstanceSingleOperationAction.java:243)
  3668.  
  3669. at org.elasticsearch.action.support.single.instance.TransportI\
  3670. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  3671. nsportInstanceSingleOperationAction.java:239)
  3672.  
  3673. at org.elasticsearch.transport.TransportRequestHandler.message\
  3674. Received(TransportRequestHandler.java:30)
  3675.  
  3676. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  3677. essageReceived(RequestHandlerRegistry.java:66)
  3678.  
  3679. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  3680. un(TcpTransport.java:1555)
  3681.  
  3682. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  3683. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  3684.  
  3685. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3686. un(AbstractRunnable.java:37)
  3687.  
  3688. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  3689. lExecutor.java:1149)
  3690.  
  3691. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  3692. olExecutor.java:624)
  3693.  
  3694. at java.lang.Thread.run(Thread.java:748)
  3695.  
  3696.  
  3697.  
  3698. "elasticsearch[fmfe16][index][T#8]" "
  3699. #410 "
  3700. daemon "
  3701. prio=5 "
  3702. os_prio=15 "
  3703. tid=0x0000000b01c99000 "
  3704. nid=0x1945b "
  3705. waiting on condition "
  3706. [0x00007fffd6361000]
  3707.  
  3708. java.lang.Thread.State: WAITING (parking)
  3709.  
  3710. at sun.misc.Unsafe.park(Native Method)
  3711.  
  3712. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  3713. rrent.locks.ReentrantLock$NonfairSync)
  3714.  
  3715. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  3716. a:175)
  3717.  
  3718. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  3719. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  3720.  
  3721. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3722. reQueued(AbstractQueuedSynchronizer.java:870)
  3723.  
  3724. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3725. re(AbstractQueuedSynchronizer.java:1199)
  3726.  
  3727. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  3728. eentrantLock.java:209)
  3729.  
  3730. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  3731. .java:285)
  3732.  
  3733. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  3734. KeyedLock.java:76)
  3735.  
  3736. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  3737. iveVersionMap.java:431)
  3738.  
  3739. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  3740. ombstones(InternalEngine.java:1661)
  3741.  
  3742. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  3743. etedTombstones(InternalEngine.java:1348)
  3744.  
  3745. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  3746. nalEngine.java:1427)
  3747.  
  3748. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  3749. ngine.java:637)
  3750.  
  3751. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  3752. a:850)
  3753.  
  3754. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  3755. etService.java:153)
  3756.  
  3757. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  3758. vice.java:81)
  3759.  
  3760. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  3761. Helper.java:74)
  3762.  
  3763. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3764. Operation(TransportUpdateAction.java:174)
  3765.  
  3766. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3767. Operation(TransportUpdateAction.java:167)
  3768.  
  3769. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  3770. Operation(TransportUpdateAction.java:67)
  3771.  
  3772. at org.elasticsearch.action.support.single.instance.TransportI\
  3773. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  3774. nsportInstanceSingleOperationAction.java:243)
  3775.  
  3776. at org.elasticsearch.action.support.single.instance.TransportI\
  3777. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  3778. nsportInstanceSingleOperationAction.java:239)
  3779.  
  3780. at org.elasticsearch.transport.TransportRequestHandler.message\
  3781. Received(TransportRequestHandler.java:30)
  3782.  
  3783. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  3784. essageReceived(RequestHandlerRegistry.java:66)
  3785.  
  3786. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  3787. un(TcpTransport.java:1555)
  3788.  
  3789. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  3790. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  3791.  
  3792. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3793. un(AbstractRunnable.java:37)
  3794.  
  3795. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  3796. lExecutor.java:1149)
  3797.  
  3798. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  3799. olExecutor.java:624)
  3800.  
  3801. at java.lang.Thread.run(Thread.java:748)
  3802.  
  3803.  
  3804.  
  3805. "elasticsearch[fmfe16][bulk][T#17]" "
  3806. #409 "
  3807. daemon "
  3808. prio=5 "
  3809. os_prio=15 "
  3810. tid=0x0000000b01c9f800 "
  3811. nid=0x19456 "
  3812. waiting on condition "
  3813. [0x00007fffc8786000]
  3814.  
  3815. java.lang.Thread.State: WAITING (parking)
  3816.  
  3817. at sun.misc.Unsafe.park(Native Method)
  3818.  
  3819. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  3820. rrent.locks.ReentrantLock$NonfairSync)
  3821.  
  3822. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  3823. a:175)
  3824.  
  3825. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  3826. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  3827.  
  3828. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3829. reQueued(AbstractQueuedSynchronizer.java:870)
  3830.  
  3831. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3832. re(AbstractQueuedSynchronizer.java:1199)
  3833.  
  3834. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  3835. eentrantLock.java:209)
  3836.  
  3837. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  3838. .java:285)
  3839.  
  3840. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  3841. KeyedLock.java:76)
  3842.  
  3843. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  3844. iveVersionMap.java:431)
  3845.  
  3846. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  3847. ombstones(InternalEngine.java:1661)
  3848.  
  3849. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  3850. etedTombstones(InternalEngine.java:1348)
  3851.  
  3852. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  3853. alEngine.java:1211)
  3854.  
  3855. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  3856. java:839)
  3857.  
  3858. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  3859. on(IndexShard.java:806)
  3860.  
  3861. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  3862. onOnPrimary(IndexShard.java:770)
  3863.  
  3864. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3865. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  3866.  
  3867. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3868. uteDeleteRequest(TransportShardBulkAction.java:154)
  3869.  
  3870. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  3871. uteBulkItemRequest(TransportShardBulkAction.java:244)
  3872.  
  3873. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  3874. ormOnPrimary(TransportShardBulkAction.java:123)
  3875.  
  3876. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  3877. dOperationOnPrimary(TransportShardBulkAction.java:110)
  3878.  
  3879. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  3880. dOperationOnPrimary(TransportShardBulkAction.java:72)
  3881.  
  3882. at org.elasticsearch.action.support.replication.TransportRepli\
  3883. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  3884. java:1034)
  3885.  
  3886. at org.elasticsearch.action.support.replication.TransportRepli\
  3887. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  3888. java:1012)
  3889.  
  3890. at org.elasticsearch.action.support.replication.ReplicationOpe\
  3891. ration.execute(ReplicationOperation.java:103)
  3892.  
  3893. at org.elasticsearch.action.support.replication.TransportRepli\
  3894. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  3895. java:359)
  3896.  
  3897. at org.elasticsearch.action.support.replication.TransportRepli\
  3898. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  3899. java:299)
  3900.  
  3901. at org.elasticsearch.action.support.replication.TransportRepli\
  3902. cationAction$1.onResponse(TransportReplicationAction.java:975)
  3903.  
  3904. at org.elasticsearch.action.support.replication.TransportRepli\
  3905. cationAction$1.onResponse(TransportReplicationAction.java:972)
  3906.  
  3907. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  3908. quire(IndexShardOperationPermits.java:238)
  3909.  
  3910. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  3911. ationPermit(IndexShard.java:2220)
  3912.  
  3913. at org.elasticsearch.action.support.replication.TransportRepli\
  3914. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  3915. ava:984)
  3916.  
  3917. at org.elasticsearch.action.support.replication.TransportRepli\
  3918. cationAction.access$500(TransportReplicationAction.java:98)
  3919.  
  3920. at org.elasticsearch.action.support.replication.TransportRepli\
  3921. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  3922. 320)
  3923.  
  3924. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3925. un(AbstractRunnable.java:37)
  3926.  
  3927. at org.elasticsearch.action.support.replication.TransportRepli\
  3928. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  3929. tReplicationAction.java:295)
  3930.  
  3931. at org.elasticsearch.action.support.replication.TransportRepli\
  3932. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  3933. tReplicationAction.java:282)
  3934.  
  3935. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  3936. essageReceived(RequestHandlerRegistry.java:66)
  3937.  
  3938. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  3939. ortService.java:656)
  3940.  
  3941. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  3942. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  3943.  
  3944. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  3945. un(AbstractRunnable.java:37)
  3946.  
  3947. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  3948. lExecutor.java:1149)
  3949.  
  3950. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  3951. olExecutor.java:624)
  3952.  
  3953. at java.lang.Thread.run(Thread.java:748)
  3954.  
  3955.  
  3956.  
  3957. "elasticsearch[fmfe16][index][T#7]" "
  3958. #408 "
  3959. daemon "
  3960. prio=5 "
  3961. os_prio=15 "
  3962. tid=0x0000000b0c6b2000 "
  3963. nid=0x19451 "
  3964. waiting on condition "
  3965. [0x00007fffd6462000]
  3966.  
  3967. java.lang.Thread.State: WAITING (parking)
  3968.  
  3969. at sun.misc.Unsafe.park(Native Method)
  3970.  
  3971. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  3972. rrent.locks.ReentrantLock$NonfairSync)
  3973.  
  3974. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  3975. a:175)
  3976.  
  3977. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  3978. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  3979.  
  3980. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3981. reQueued(AbstractQueuedSynchronizer.java:870)
  3982.  
  3983. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  3984. re(AbstractQueuedSynchronizer.java:1199)
  3985.  
  3986. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  3987. eentrantLock.java:209)
  3988.  
  3989. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  3990. .java:285)
  3991.  
  3992. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  3993. KeyedLock.java:76)
  3994.  
  3995. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  3996. iveVersionMap.java:431)
  3997.  
  3998. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  3999. ombstones(InternalEngine.java:1661)
  4000.  
  4001. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  4002. etedTombstones(InternalEngine.java:1348)
  4003.  
  4004. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  4005. nalEngine.java:1427)
  4006.  
  4007. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  4008. ngine.java:637)
  4009.  
  4010. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  4011. a:850)
  4012.  
  4013. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  4014. etService.java:153)
  4015.  
  4016. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  4017. vice.java:81)
  4018.  
  4019. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  4020. Helper.java:74)
  4021.  
  4022. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4023. Operation(TransportUpdateAction.java:174)
  4024.  
  4025. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4026. Operation(TransportUpdateAction.java:167)
  4027.  
  4028. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4029. Operation(TransportUpdateAction.java:67)
  4030.  
  4031. at org.elasticsearch.action.support.single.instance.TransportI\
  4032. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  4033. nsportInstanceSingleOperationAction.java:243)
  4034.  
  4035. at org.elasticsearch.action.support.single.instance.TransportI\
  4036. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  4037. nsportInstanceSingleOperationAction.java:239)
  4038.  
  4039. at org.elasticsearch.transport.TransportRequestHandler.message\
  4040. Received(TransportRequestHandler.java:30)
  4041.  
  4042. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  4043. essageReceived(RequestHandlerRegistry.java:66)
  4044.  
  4045. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  4046. un(TcpTransport.java:1555)
  4047.  
  4048. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  4049. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  4050.  
  4051. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4052. un(AbstractRunnable.java:37)
  4053.  
  4054. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  4055. lExecutor.java:1149)
  4056.  
  4057. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  4058. olExecutor.java:624)
  4059.  
  4060. at java.lang.Thread.run(Thread.java:748)
  4061.  
  4062.  
  4063.  
  4064. "elasticsearch[fmfe16][bulk][T#16]" "
  4065. #407 "
  4066. daemon "
  4067. prio=5 "
  4068. os_prio=15 "
  4069. tid=0x0000000afde90800 "
  4070. nid=0x1944f "
  4071. waiting on condition "
  4072. [0x00007fffc9291000]
  4073.  
  4074. java.lang.Thread.State: WAITING (parking)
  4075.  
  4076. at sun.misc.Unsafe.park(Native Method)
  4077.  
  4078. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  4079. rrent.locks.ReentrantLock$NonfairSync)
  4080.  
  4081. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  4082. a:175)
  4083.  
  4084. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  4085. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  4086.  
  4087. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4088. reQueued(AbstractQueuedSynchronizer.java:870)
  4089.  
  4090. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4091. re(AbstractQueuedSynchronizer.java:1199)
  4092.  
  4093. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  4094. eentrantLock.java:209)
  4095.  
  4096. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  4097. .java:285)
  4098.  
  4099. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  4100. KeyedLock.java:76)
  4101.  
  4102. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  4103. iveVersionMap.java:431)
  4104.  
  4105. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  4106. ombstones(InternalEngine.java:1661)
  4107.  
  4108. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  4109. etedTombstones(InternalEngine.java:1348)
  4110.  
  4111. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  4112. nalEngine.java:1427)
  4113.  
  4114. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  4115. ngine.java:637)
  4116.  
  4117. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  4118. a:850)
  4119.  
  4120. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  4121. etService.java:153)
  4122.  
  4123. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  4124. vice.java:81)
  4125.  
  4126. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  4127. Helper.java:74)
  4128.  
  4129. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4130. uteUpdateRequestOnce(TransportShardBulkAction.java:343)
  4131.  
  4132. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4133. uteUpdateRequest(TransportShardBulkAction.java:406)
  4134.  
  4135. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4136. uteBulkItemRequest(TransportShardBulkAction.java:240)
  4137.  
  4138. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  4139. ormOnPrimary(TransportShardBulkAction.java:123)
  4140.  
  4141. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4142. dOperationOnPrimary(TransportShardBulkAction.java:110)
  4143.  
  4144. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4145. dOperationOnPrimary(TransportShardBulkAction.java:72)
  4146.  
  4147. at org.elasticsearch.action.support.replication.TransportRepli\
  4148. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4149. java:1034)
  4150.  
  4151. at org.elasticsearch.action.support.replication.TransportRepli\
  4152. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4153. java:1012)
  4154.  
  4155. at org.elasticsearch.action.support.replication.ReplicationOpe\
  4156. ration.execute(ReplicationOperation.java:103)
  4157.  
  4158. at org.elasticsearch.action.support.replication.TransportRepli\
  4159. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4160. java:359)
  4161.  
  4162. at org.elasticsearch.action.support.replication.TransportRepli\
  4163. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4164. java:299)
  4165.  
  4166. at org.elasticsearch.action.support.replication.TransportRepli\
  4167. cationAction$1.onResponse(TransportReplicationAction.java:975)
  4168.  
  4169. at org.elasticsearch.action.support.replication.TransportRepli\
  4170. cationAction$1.onResponse(TransportReplicationAction.java:972)
  4171.  
  4172. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  4173. quire(IndexShardOperationPermits.java:238)
  4174.  
  4175. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  4176. ationPermit(IndexShard.java:2220)
  4177.  
  4178. at org.elasticsearch.action.support.replication.TransportRepli\
  4179. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  4180. ava:984)
  4181.  
  4182. at org.elasticsearch.action.support.replication.TransportRepli\
  4183. cationAction.access$500(TransportReplicationAction.java:98)
  4184.  
  4185. at org.elasticsearch.action.support.replication.TransportRepli\
  4186. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  4187. 320)
  4188.  
  4189. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4190. un(AbstractRunnable.java:37)
  4191.  
  4192. at org.elasticsearch.action.support.replication.TransportRepli\
  4193. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  4194. tReplicationAction.java:295)
  4195.  
  4196. at org.elasticsearch.action.support.replication.TransportRepli\
  4197. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  4198. tReplicationAction.java:282)
  4199.  
  4200. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  4201. essageReceived(RequestHandlerRegistry.java:66)
  4202.  
  4203. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  4204. ortService.java:656)
  4205.  
  4206. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  4207. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  4208.  
  4209. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4210. un(AbstractRunnable.java:37)
  4211.  
  4212. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  4213. lExecutor.java:1149)
  4214.  
  4215. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  4216. olExecutor.java:624)
  4217.  
  4218. at java.lang.Thread.run(Thread.java:748)
  4219.  
  4220.  
  4221.  
  4222. "elasticsearch[fmfe16][index][T#6]" "
  4223. #406 "
  4224. daemon "
  4225. prio=5 "
  4226. os_prio=15 "
  4227. tid=0x0000000b046bd000 "
  4228. nid=0x1944a "
  4229. waiting on condition "
  4230. [0x00007fffca2a1000]
  4231.  
  4232. java.lang.Thread.State: WAITING (parking)
  4233.  
  4234. at sun.misc.Unsafe.park(Native Method)
  4235.  
  4236. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  4237. rrent.locks.ReentrantLock$NonfairSync)
  4238.  
  4239. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  4240. a:175)
  4241.  
  4242. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  4243. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  4244.  
  4245. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4246. reQueued(AbstractQueuedSynchronizer.java:870)
  4247.  
  4248. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4249. re(AbstractQueuedSynchronizer.java:1199)
  4250.  
  4251. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  4252. eentrantLock.java:209)
  4253.  
  4254. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  4255. .java:285)
  4256.  
  4257. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  4258. KeyedLock.java:76)
  4259.  
  4260. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  4261. iveVersionMap.java:431)
  4262.  
  4263. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  4264. ombstones(InternalEngine.java:1661)
  4265.  
  4266. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  4267. etedTombstones(InternalEngine.java:1348)
  4268.  
  4269. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  4270. nalEngine.java:1427)
  4271.  
  4272. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  4273. ngine.java:637)
  4274.  
  4275. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  4276. a:850)
  4277.  
  4278. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  4279. etService.java:153)
  4280.  
  4281. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  4282. vice.java:81)
  4283.  
  4284. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  4285. Helper.java:74)
  4286.  
  4287. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4288. Operation(TransportUpdateAction.java:174)
  4289.  
  4290. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4291. Operation(TransportUpdateAction.java:167)
  4292.  
  4293. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4294. Operation(TransportUpdateAction.java:67)
  4295.  
  4296. at org.elasticsearch.action.support.single.instance.TransportI\
  4297. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  4298. nsportInstanceSingleOperationAction.java:243)
  4299.  
  4300. at org.elasticsearch.action.support.single.instance.TransportI\
  4301. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  4302. nsportInstanceSingleOperationAction.java:239)
  4303.  
  4304. at org.elasticsearch.transport.TransportRequestHandler.message\
  4305. Received(TransportRequestHandler.java:30)
  4306.  
  4307. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  4308. essageReceived(RequestHandlerRegistry.java:66)
  4309.  
  4310. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  4311. un(TcpTransport.java:1555)
  4312.  
  4313. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  4314. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  4315.  
  4316. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4317. un(AbstractRunnable.java:37)
  4318.  
  4319. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  4320. lExecutor.java:1149)
  4321.  
  4322. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  4323. olExecutor.java:624)
  4324.  
  4325. at java.lang.Thread.run(Thread.java:748)
  4326.  
  4327.  
  4328.  
  4329. "elasticsearch[fmfe16][bulk][T#15]" "
  4330. #405 "
  4331. daemon "
  4332. prio=5 "
  4333. os_prio=15 "
  4334. tid=0x0000000b00aa6800 "
  4335. nid=0x193b6 "
  4336. waiting on condition "
  4337. [0x00007fffca3a2000]
  4338.  
  4339. java.lang.Thread.State: WAITING (parking)
  4340.  
  4341. at sun.misc.Unsafe.park(Native Method)
  4342.  
  4343. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  4344. rrent.locks.ReentrantLock$NonfairSync)
  4345.  
  4346. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  4347. a:175)
  4348.  
  4349. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  4350. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  4351.  
  4352. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4353. reQueued(AbstractQueuedSynchronizer.java:870)
  4354.  
  4355. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4356. re(AbstractQueuedSynchronizer.java:1199)
  4357.  
  4358. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  4359. eentrantLock.java:209)
  4360.  
  4361. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  4362. .java:285)
  4363.  
  4364. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  4365. KeyedLock.java:76)
  4366.  
  4367. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  4368. iveVersionMap.java:431)
  4369.  
  4370. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  4371. ombstones(InternalEngine.java:1661)
  4372.  
  4373. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  4374. etedTombstones(InternalEngine.java:1348)
  4375.  
  4376. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  4377. alEngine.java:1211)
  4378.  
  4379. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  4380. java:839)
  4381.  
  4382. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  4383. on(IndexShard.java:806)
  4384.  
  4385. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  4386. onOnPrimary(IndexShard.java:770)
  4387.  
  4388. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4389. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  4390.  
  4391. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4392. uteDeleteRequest(TransportShardBulkAction.java:154)
  4393.  
  4394. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4395. uteBulkItemRequest(TransportShardBulkAction.java:244)
  4396.  
  4397. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  4398. ormOnPrimary(TransportShardBulkAction.java:123)
  4399.  
  4400. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4401. dOperationOnPrimary(TransportShardBulkAction.java:110)
  4402.  
  4403. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4404. dOperationOnPrimary(TransportShardBulkAction.java:72)
  4405.  
  4406. at org.elasticsearch.action.support.replication.TransportRepli\
  4407. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4408. java:1034)
  4409.  
  4410. at org.elasticsearch.action.support.replication.TransportRepli\
  4411. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4412. java:1012)
  4413.  
  4414. at org.elasticsearch.action.support.replication.ReplicationOpe\
  4415. ration.execute(ReplicationOperation.java:103)
  4416.  
  4417. at org.elasticsearch.action.support.replication.TransportRepli\
  4418. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4419. java:359)
  4420.  
  4421. at org.elasticsearch.action.support.replication.TransportRepli\
  4422. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4423. java:299)
  4424.  
  4425. at org.elasticsearch.action.support.replication.TransportRepli\
  4426. cationAction$1.onResponse(TransportReplicationAction.java:975)
  4427.  
  4428. at org.elasticsearch.action.support.replication.TransportRepli\
  4429. cationAction$1.onResponse(TransportReplicationAction.java:972)
  4430.  
  4431. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  4432. quire(IndexShardOperationPermits.java:238)
  4433.  
  4434. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  4435. ationPermit(IndexShard.java:2220)
  4436.  
  4437. at org.elasticsearch.action.support.replication.TransportRepli\
  4438. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  4439. ava:984)
  4440.  
  4441. at org.elasticsearch.action.support.replication.TransportRepli\
  4442. cationAction.access$500(TransportReplicationAction.java:98)
  4443.  
  4444. at org.elasticsearch.action.support.replication.TransportRepli\
  4445. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  4446. 320)
  4447.  
  4448. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4449. un(AbstractRunnable.java:37)
  4450.  
  4451. at org.elasticsearch.action.support.replication.TransportRepli\
  4452. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  4453. tReplicationAction.java:295)
  4454.  
  4455. at org.elasticsearch.action.support.replication.TransportRepli\
  4456. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  4457. tReplicationAction.java:282)
  4458.  
  4459. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  4460. essageReceived(RequestHandlerRegistry.java:66)
  4461.  
  4462. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  4463. ortService.java:656)
  4464.  
  4465. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  4466. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  4467.  
  4468. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4469. un(AbstractRunnable.java:37)
  4470.  
  4471. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  4472. lExecutor.java:1149)
  4473.  
  4474. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  4475. olExecutor.java:624)
  4476.  
  4477. at java.lang.Thread.run(Thread.java:748)
  4478.  
  4479.  
  4480.  
  4481. "elasticsearch[fmfe16][bulk][T#14]" "
  4482. #403 "
  4483. daemon "
  4484. prio=5 "
  4485. os_prio=15 "
  4486. tid=0x0000000b00aa6000 "
  4487. nid=0x191ab "
  4488. waiting on condition "
  4489. [0x00007fffca8a7000]
  4490.  
  4491. java.lang.Thread.State: WAITING (parking)
  4492.  
  4493. at sun.misc.Unsafe.park(Native Method)
  4494.  
  4495. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  4496. rrent.locks.ReentrantLock$NonfairSync)
  4497.  
  4498. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  4499. a:175)
  4500.  
  4501. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  4502. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  4503.  
  4504. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4505. reQueued(AbstractQueuedSynchronizer.java:870)
  4506.  
  4507. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4508. re(AbstractQueuedSynchronizer.java:1199)
  4509.  
  4510. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  4511. eentrantLock.java:209)
  4512.  
  4513. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  4514. .java:285)
  4515.  
  4516. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  4517. KeyedLock.java:76)
  4518.  
  4519. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  4520. iveVersionMap.java:431)
  4521.  
  4522. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  4523. ombstones(InternalEngine.java:1661)
  4524.  
  4525. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  4526. etedTombstones(InternalEngine.java:1348)
  4527.  
  4528. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  4529. alEngine.java:1211)
  4530.  
  4531. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  4532. java:839)
  4533.  
  4534. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  4535. on(IndexShard.java:806)
  4536.  
  4537. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  4538. onOnPrimary(IndexShard.java:770)
  4539.  
  4540. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4541. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  4542.  
  4543. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4544. uteDeleteRequest(TransportShardBulkAction.java:154)
  4545.  
  4546. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4547. uteBulkItemRequest(TransportShardBulkAction.java:244)
  4548.  
  4549. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  4550. ormOnPrimary(TransportShardBulkAction.java:123)
  4551.  
  4552. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4553. dOperationOnPrimary(TransportShardBulkAction.java:110)
  4554.  
  4555. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4556. dOperationOnPrimary(TransportShardBulkAction.java:72)
  4557.  
  4558. at org.elasticsearch.action.support.replication.TransportRepli\
  4559. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4560. java:1034)
  4561.  
  4562. at org.elasticsearch.action.support.replication.TransportRepli\
  4563. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4564. java:1012)
  4565.  
  4566. at org.elasticsearch.action.support.replication.ReplicationOpe\
  4567. ration.execute(ReplicationOperation.java:103)
  4568.  
  4569. at org.elasticsearch.action.support.replication.TransportRepli\
  4570. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4571. java:359)
  4572.  
  4573. at org.elasticsearch.action.support.replication.TransportRepli\
  4574. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4575. java:299)
  4576.  
  4577. at org.elasticsearch.action.support.replication.TransportRepli\
  4578. cationAction$1.onResponse(TransportReplicationAction.java:975)
  4579.  
  4580. at org.elasticsearch.action.support.replication.TransportRepli\
  4581. cationAction$1.onResponse(TransportReplicationAction.java:972)
  4582.  
  4583. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  4584. quire(IndexShardOperationPermits.java:238)
  4585.  
  4586. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  4587. ationPermit(IndexShard.java:2220)
  4588.  
  4589. at org.elasticsearch.action.support.replication.TransportRepli\
  4590. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  4591. ava:984)
  4592.  
  4593. at org.elasticsearch.action.support.replication.TransportRepli\
  4594. cationAction.access$500(TransportReplicationAction.java:98)
  4595.  
  4596. at org.elasticsearch.action.support.replication.TransportRepli\
  4597. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  4598. 320)
  4599.  
  4600. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4601. un(AbstractRunnable.java:37)
  4602.  
  4603. at org.elasticsearch.action.support.replication.TransportRepli\
  4604. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  4605. tReplicationAction.java:295)
  4606.  
  4607. at org.elasticsearch.action.support.replication.TransportRepli\
  4608. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  4609. tReplicationAction.java:282)
  4610.  
  4611. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  4612. essageReceived(RequestHandlerRegistry.java:66)
  4613.  
  4614. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  4615. ortService.java:656)
  4616.  
  4617. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  4618. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  4619.  
  4620. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4621. un(AbstractRunnable.java:37)
  4622.  
  4623. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  4624. lExecutor.java:1149)
  4625.  
  4626. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  4627. olExecutor.java:624)
  4628.  
  4629. at java.lang.Thread.run(Thread.java:748)
  4630.  
  4631.  
  4632.  
  4633. "elasticsearch[fmfe16][index][T#5]" "
  4634. #401 "
  4635. daemon "
  4636. prio=5 "
  4637. os_prio=15 "
  4638. tid=0x0000000b0cec5800 "
  4639. nid=0x1902c "
  4640. waiting on condition "
  4641. [0x00007fffd5957000]
  4642.  
  4643. java.lang.Thread.State: WAITING (parking)
  4644.  
  4645. at sun.misc.Unsafe.park(Native Method)
  4646.  
  4647. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  4648. rrent.locks.ReentrantLock$NonfairSync)
  4649.  
  4650. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  4651. a:175)
  4652.  
  4653. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  4654. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  4655.  
  4656. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4657. reQueued(AbstractQueuedSynchronizer.java:870)
  4658.  
  4659. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4660. re(AbstractQueuedSynchronizer.java:1199)
  4661.  
  4662. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  4663. eentrantLock.java:209)
  4664.  
  4665. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  4666. .java:285)
  4667.  
  4668. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  4669. KeyedLock.java:76)
  4670.  
  4671. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  4672. iveVersionMap.java:431)
  4673.  
  4674. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  4675. ombstones(InternalEngine.java:1661)
  4676.  
  4677. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  4678. etedTombstones(InternalEngine.java:1348)
  4679.  
  4680. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  4681. nalEngine.java:1427)
  4682.  
  4683. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  4684. ngine.java:637)
  4685.  
  4686. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  4687. a:850)
  4688.  
  4689. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  4690. etService.java:153)
  4691.  
  4692. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  4693. vice.java:81)
  4694.  
  4695. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  4696. Helper.java:74)
  4697.  
  4698. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4699. Operation(TransportUpdateAction.java:174)
  4700.  
  4701. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4702. Operation(TransportUpdateAction.java:167)
  4703.  
  4704. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  4705. Operation(TransportUpdateAction.java:67)
  4706.  
  4707. at org.elasticsearch.action.support.single.instance.TransportI\
  4708. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  4709. nsportInstanceSingleOperationAction.java:243)
  4710.  
  4711. at org.elasticsearch.action.support.single.instance.TransportI\
  4712. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  4713. nsportInstanceSingleOperationAction.java:239)
  4714.  
  4715. at org.elasticsearch.transport.TransportRequestHandler.message\
  4716. Received(TransportRequestHandler.java:30)
  4717.  
  4718. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  4719. essageReceived(RequestHandlerRegistry.java:66)
  4720.  
  4721. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  4722. un(TcpTransport.java:1555)
  4723.  
  4724. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  4725. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  4726.  
  4727. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4728. un(AbstractRunnable.java:37)
  4729.  
  4730. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  4731. lExecutor.java:1149)
  4732.  
  4733. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  4734. olExecutor.java:624)
  4735.  
  4736. at java.lang.Thread.run(Thread.java:748)
  4737.  
  4738.  
  4739.  
  4740. "elasticsearch[fmfe16][bulk][T#13]" "
  4741. #400 "
  4742. daemon "
  4743. prio=5 "
  4744. os_prio=15 "
  4745. tid=0x0000000b0132f800 "
  4746. nid=0x18fdb "
  4747. waiting on condition "
  4748. [0x00007fffc8685000]
  4749.  
  4750. java.lang.Thread.State: WAITING (parking)
  4751.  
  4752. at sun.misc.Unsafe.park(Native Method)
  4753.  
  4754. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  4755. rrent.locks.ReentrantLock$NonfairSync)
  4756.  
  4757. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  4758. a:175)
  4759.  
  4760. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  4761. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  4762.  
  4763. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4764. reQueued(AbstractQueuedSynchronizer.java:870)
  4765.  
  4766. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4767. re(AbstractQueuedSynchronizer.java:1199)
  4768.  
  4769. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  4770. eentrantLock.java:209)
  4771.  
  4772. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  4773. .java:285)
  4774.  
  4775. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  4776. KeyedLock.java:76)
  4777.  
  4778. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  4779. iveVersionMap.java:431)
  4780.  
  4781. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  4782. ombstones(InternalEngine.java:1661)
  4783.  
  4784. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  4785. etedTombstones(InternalEngine.java:1348)
  4786.  
  4787. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  4788. nalEngine.java:1427)
  4789.  
  4790. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  4791. ngine.java:637)
  4792.  
  4793. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  4794. a:850)
  4795.  
  4796. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  4797. etService.java:153)
  4798.  
  4799. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  4800. vice.java:81)
  4801.  
  4802. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  4803. Helper.java:74)
  4804.  
  4805. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4806. uteUpdateRequestOnce(TransportShardBulkAction.java:343)
  4807.  
  4808. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4809. uteUpdateRequest(TransportShardBulkAction.java:406)
  4810.  
  4811. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4812. uteBulkItemRequest(TransportShardBulkAction.java:240)
  4813.  
  4814. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  4815. ormOnPrimary(TransportShardBulkAction.java:123)
  4816.  
  4817. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4818. dOperationOnPrimary(TransportShardBulkAction.java:110)
  4819.  
  4820. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4821. dOperationOnPrimary(TransportShardBulkAction.java:72)
  4822.  
  4823. at org.elasticsearch.action.support.replication.TransportRepli\
  4824. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4825. java:1034)
  4826.  
  4827. at org.elasticsearch.action.support.replication.TransportRepli\
  4828. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4829. java:1012)
  4830.  
  4831. at org.elasticsearch.action.support.replication.ReplicationOpe\
  4832. ration.execute(ReplicationOperation.java:103)
  4833.  
  4834. at org.elasticsearch.action.support.replication.TransportRepli\
  4835. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4836. java:359)
  4837.  
  4838. at org.elasticsearch.action.support.replication.TransportRepli\
  4839. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4840. java:299)
  4841.  
  4842. at org.elasticsearch.action.support.replication.TransportRepli\
  4843. cationAction$1.onResponse(TransportReplicationAction.java:975)
  4844.  
  4845. at org.elasticsearch.action.support.replication.TransportRepli\
  4846. cationAction$1.onResponse(TransportReplicationAction.java:972)
  4847.  
  4848. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  4849. quire(IndexShardOperationPermits.java:238)
  4850.  
  4851. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  4852. ationPermit(IndexShard.java:2220)
  4853.  
  4854. at org.elasticsearch.action.support.replication.TransportRepli\
  4855. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  4856. ava:984)
  4857.  
  4858. at org.elasticsearch.action.support.replication.TransportRepli\
  4859. cationAction.access$500(TransportReplicationAction.java:98)
  4860.  
  4861. at org.elasticsearch.action.support.replication.TransportRepli\
  4862. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  4863. 320)
  4864.  
  4865. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4866. un(AbstractRunnable.java:37)
  4867.  
  4868. at org.elasticsearch.action.support.replication.TransportRepli\
  4869. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  4870. tReplicationAction.java:295)
  4871.  
  4872. at org.elasticsearch.action.support.replication.TransportRepli\
  4873. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  4874. tReplicationAction.java:282)
  4875.  
  4876. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  4877. essageReceived(RequestHandlerRegistry.java:66)
  4878.  
  4879. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  4880. ortService.java:656)
  4881.  
  4882. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  4883. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  4884.  
  4885. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  4886. un(AbstractRunnable.java:37)
  4887.  
  4888. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  4889. lExecutor.java:1149)
  4890.  
  4891. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  4892. olExecutor.java:624)
  4893.  
  4894. at java.lang.Thread.run(Thread.java:748)
  4895.  
  4896.  
  4897.  
  4898. "elasticsearch[fmfe16][bulk][T#12]" "
  4899. #399 "
  4900. daemon "
  4901. prio=5 "
  4902. os_prio=15 "
  4903. tid=0x0000000b00aa5000 "
  4904. nid=0x187a1 "
  4905. waiting on condition "
  4906. [0x00007fffca4a3000]
  4907.  
  4908. java.lang.Thread.State: WAITING (parking)
  4909.  
  4910. at sun.misc.Unsafe.park(Native Method)
  4911.  
  4912. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  4913. rrent.locks.ReentrantLock$NonfairSync)
  4914.  
  4915. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  4916. a:175)
  4917.  
  4918. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  4919. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  4920.  
  4921. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4922. reQueued(AbstractQueuedSynchronizer.java:870)
  4923.  
  4924. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  4925. re(AbstractQueuedSynchronizer.java:1199)
  4926.  
  4927. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  4928. eentrantLock.java:209)
  4929.  
  4930. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  4931. .java:285)
  4932.  
  4933. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  4934. KeyedLock.java:76)
  4935.  
  4936. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  4937. iveVersionMap.java:431)
  4938.  
  4939. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  4940. ombstones(InternalEngine.java:1661)
  4941.  
  4942. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  4943. etedTombstones(InternalEngine.java:1348)
  4944.  
  4945. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  4946. alEngine.java:1211)
  4947.  
  4948. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  4949. java:839)
  4950.  
  4951. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  4952. on(IndexShard.java:806)
  4953.  
  4954. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  4955. onOnPrimary(IndexShard.java:770)
  4956.  
  4957. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4958. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  4959.  
  4960. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4961. uteDeleteRequest(TransportShardBulkAction.java:154)
  4962.  
  4963. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  4964. uteBulkItemRequest(TransportShardBulkAction.java:244)
  4965.  
  4966. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  4967. ormOnPrimary(TransportShardBulkAction.java:123)
  4968.  
  4969. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4970. dOperationOnPrimary(TransportShardBulkAction.java:110)
  4971.  
  4972. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  4973. dOperationOnPrimary(TransportShardBulkAction.java:72)
  4974.  
  4975. at org.elasticsearch.action.support.replication.TransportRepli\
  4976. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4977. java:1034)
  4978.  
  4979. at org.elasticsearch.action.support.replication.TransportRepli\
  4980. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  4981. java:1012)
  4982.  
  4983. at org.elasticsearch.action.support.replication.ReplicationOpe\
  4984. ration.execute(ReplicationOperation.java:103)
  4985.  
  4986. at org.elasticsearch.action.support.replication.TransportRepli\
  4987. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4988. java:359)
  4989.  
  4990. at org.elasticsearch.action.support.replication.TransportRepli\
  4991. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  4992. java:299)
  4993.  
  4994. at org.elasticsearch.action.support.replication.TransportRepli\
  4995. cationAction$1.onResponse(TransportReplicationAction.java:975)
  4996.  
  4997. at org.elasticsearch.action.support.replication.TransportRepli\
  4998. cationAction$1.onResponse(TransportReplicationAction.java:972)
  4999.  
  5000. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  5001. quire(IndexShardOperationPermits.java:238)
  5002.  
  5003. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  5004. ationPermit(IndexShard.java:2220)
  5005.  
  5006. at org.elasticsearch.action.support.replication.TransportRepli\
  5007. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  5008. ava:984)
  5009.  
  5010. at org.elasticsearch.action.support.replication.TransportRepli\
  5011. cationAction.access$500(TransportReplicationAction.java:98)
  5012.  
  5013. at org.elasticsearch.action.support.replication.TransportRepli\
  5014. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  5015. 320)
  5016.  
  5017. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5018. un(AbstractRunnable.java:37)
  5019.  
  5020. at org.elasticsearch.action.support.replication.TransportRepli\
  5021. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5022. tReplicationAction.java:295)
  5023.  
  5024. at org.elasticsearch.action.support.replication.TransportRepli\
  5025. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5026. tReplicationAction.java:282)
  5027.  
  5028. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  5029. essageReceived(RequestHandlerRegistry.java:66)
  5030.  
  5031. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  5032. ortService.java:656)
  5033.  
  5034. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  5035. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  5036.  
  5037. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5038. un(AbstractRunnable.java:37)
  5039.  
  5040. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  5041. lExecutor.java:1149)
  5042.  
  5043. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  5044. olExecutor.java:624)
  5045.  
  5046. at java.lang.Thread.run(Thread.java:748)
  5047.  
  5048.  
  5049.  
  5050. "elasticsearch[fmfe16][bulk][T#11]" "
  5051. #393 "
  5052. daemon "
  5053. prio=5 "
  5054. os_prio=15 "
  5055. tid=0x0000000b00aa4800 "
  5056. nid=0x190eb "
  5057. waiting on condition "
  5058. [0x00007fffd9492000]
  5059.  
  5060. java.lang.Thread.State: WAITING (parking)
  5061.  
  5062. at sun.misc.Unsafe.park(Native Method)
  5063.  
  5064. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  5065. rrent.locks.ReentrantLock$NonfairSync)
  5066.  
  5067. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  5068. a:175)
  5069.  
  5070. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  5071. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  5072.  
  5073. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5074. reQueued(AbstractQueuedSynchronizer.java:870)
  5075.  
  5076. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5077. re(AbstractQueuedSynchronizer.java:1199)
  5078.  
  5079. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  5080. eentrantLock.java:209)
  5081.  
  5082. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  5083. .java:285)
  5084.  
  5085. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  5086. KeyedLock.java:76)
  5087.  
  5088. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  5089. iveVersionMap.java:431)
  5090.  
  5091. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  5092. ombstones(InternalEngine.java:1661)
  5093.  
  5094. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  5095. etedTombstones(InternalEngine.java:1348)
  5096.  
  5097. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  5098. alEngine.java:1211)
  5099.  
  5100. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  5101. java:839)
  5102.  
  5103. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  5104. on(IndexShard.java:806)
  5105.  
  5106. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  5107. onOnPrimary(IndexShard.java:770)
  5108.  
  5109. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5110. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  5111.  
  5112. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5113. uteDeleteRequest(TransportShardBulkAction.java:154)
  5114.  
  5115. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5116. uteBulkItemRequest(TransportShardBulkAction.java:244)
  5117.  
  5118. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  5119. ormOnPrimary(TransportShardBulkAction.java:123)
  5120.  
  5121. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5122. dOperationOnPrimary(TransportShardBulkAction.java:110)
  5123.  
  5124. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5125. dOperationOnPrimary(TransportShardBulkAction.java:72)
  5126.  
  5127. at org.elasticsearch.action.support.replication.TransportRepli\
  5128. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5129. java:1034)
  5130.  
  5131. at org.elasticsearch.action.support.replication.TransportRepli\
  5132. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5133. java:1012)
  5134.  
  5135. at org.elasticsearch.action.support.replication.ReplicationOpe\
  5136. ration.execute(ReplicationOperation.java:103)
  5137.  
  5138. at org.elasticsearch.action.support.replication.TransportRepli\
  5139. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5140. java:359)
  5141.  
  5142. at org.elasticsearch.action.support.replication.TransportRepli\
  5143. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5144. java:299)
  5145.  
  5146. at org.elasticsearch.action.support.replication.TransportRepli\
  5147. cationAction$1.onResponse(TransportReplicationAction.java:975)
  5148.  
  5149. at org.elasticsearch.action.support.replication.TransportRepli\
  5150. cationAction$1.onResponse(TransportReplicationAction.java:972)
  5151.  
  5152. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  5153. quire(IndexShardOperationPermits.java:238)
  5154.  
  5155. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  5156. ationPermit(IndexShard.java:2220)
  5157.  
  5158. at org.elasticsearch.action.support.replication.TransportRepli\
  5159. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  5160. ava:984)
  5161.  
  5162. at org.elasticsearch.action.support.replication.TransportRepli\
  5163. cationAction.access$500(TransportReplicationAction.java:98)
  5164.  
  5165. at org.elasticsearch.action.support.replication.TransportRepli\
  5166. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  5167. 320)
  5168.  
  5169. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5170. un(AbstractRunnable.java:37)
  5171.  
  5172. at org.elasticsearch.action.support.replication.TransportRepli\
  5173. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5174. tReplicationAction.java:295)
  5175.  
  5176. at org.elasticsearch.action.support.replication.TransportRepli\
  5177. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5178. tReplicationAction.java:282)
  5179.  
  5180. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  5181. essageReceived(RequestHandlerRegistry.java:66)
  5182.  
  5183. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  5184. ortService.java:656)
  5185.  
  5186. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  5187. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  5188.  
  5189. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5190. un(AbstractRunnable.java:37)
  5191.  
  5192. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  5193. lExecutor.java:1149)
  5194.  
  5195. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  5196. olExecutor.java:624)
  5197.  
  5198. at java.lang.Thread.run(Thread.java:748)
  5199.  
  5200.  
  5201.  
  5202. "elasticsearch[fmfe16][index][T#4]" "
  5203. #392 "
  5204. daemon "
  5205. prio=5 "
  5206. os_prio=15 "
  5207. tid=0x0000000b0c66f800 "
  5208. nid=0x190c9 "
  5209. waiting on condition "
  5210. [0x00007fffd7674000]
  5211.  
  5212. java.lang.Thread.State: WAITING (parking)
  5213.  
  5214. at sun.misc.Unsafe.park(Native Method)
  5215.  
  5216. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  5217. rrent.locks.ReentrantLock$NonfairSync)
  5218.  
  5219. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  5220. a:175)
  5221.  
  5222. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  5223. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  5224.  
  5225. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5226. reQueued(AbstractQueuedSynchronizer.java:870)
  5227.  
  5228. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5229. re(AbstractQueuedSynchronizer.java:1199)
  5230.  
  5231. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  5232. eentrantLock.java:209)
  5233.  
  5234. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  5235. .java:285)
  5236.  
  5237. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  5238. KeyedLock.java:76)
  5239.  
  5240. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  5241. iveVersionMap.java:431)
  5242.  
  5243. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  5244. ombstones(InternalEngine.java:1661)
  5245.  
  5246. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  5247. etedTombstones(InternalEngine.java:1348)
  5248.  
  5249. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  5250. nalEngine.java:1427)
  5251.  
  5252. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  5253. ngine.java:637)
  5254.  
  5255. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  5256. a:850)
  5257.  
  5258. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  5259. etService.java:153)
  5260.  
  5261. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  5262. vice.java:81)
  5263.  
  5264. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  5265. Helper.java:74)
  5266.  
  5267. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  5268. Operation(TransportUpdateAction.java:174)
  5269.  
  5270. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  5271. Operation(TransportUpdateAction.java:167)
  5272.  
  5273. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  5274. Operation(TransportUpdateAction.java:67)
  5275.  
  5276. at org.elasticsearch.action.support.single.instance.TransportI\
  5277. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  5278. nsportInstanceSingleOperationAction.java:243)
  5279.  
  5280. at org.elasticsearch.action.support.single.instance.TransportI\
  5281. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  5282. nsportInstanceSingleOperationAction.java:239)
  5283.  
  5284. at org.elasticsearch.transport.TransportRequestHandler.message\
  5285. Received(TransportRequestHandler.java:30)
  5286.  
  5287. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  5288. essageReceived(RequestHandlerRegistry.java:66)
  5289.  
  5290. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  5291. un(TcpTransport.java:1555)
  5292.  
  5293. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  5294. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  5295.  
  5296. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5297. un(AbstractRunnable.java:37)
  5298.  
  5299. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  5300. lExecutor.java:1149)
  5301.  
  5302. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  5303. olExecutor.java:624)
  5304.  
  5305. at java.lang.Thread.run(Thread.java:748)
  5306.  
  5307.  
  5308.  
  5309. "elasticsearch[fmfe16][bulk][T#10]" "
  5310. #389 "
  5311. daemon "
  5312. prio=5 "
  5313. os_prio=15 "
  5314. tid=0x0000000b1131d000 "
  5315. nid=0x18f52 "
  5316. waiting on condition "
  5317. [0x00007fffd8280000]
  5318.  
  5319. java.lang.Thread.State: WAITING (parking)
  5320.  
  5321. at sun.misc.Unsafe.park(Native Method)
  5322.  
  5323. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  5324. rrent.locks.ReentrantLock$NonfairSync)
  5325.  
  5326. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  5327. a:175)
  5328.  
  5329. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  5330. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  5331.  
  5332. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5333. reQueued(AbstractQueuedSynchronizer.java:870)
  5334.  
  5335. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5336. re(AbstractQueuedSynchronizer.java:1199)
  5337.  
  5338. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  5339. eentrantLock.java:209)
  5340.  
  5341. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  5342. .java:285)
  5343.  
  5344. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  5345. KeyedLock.java:76)
  5346.  
  5347. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  5348. iveVersionMap.java:431)
  5349.  
  5350. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  5351. ombstones(InternalEngine.java:1661)
  5352.  
  5353. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  5354. etedTombstones(InternalEngine.java:1348)
  5355.  
  5356. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  5357. nalEngine.java:1427)
  5358.  
  5359. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  5360. ngine.java:637)
  5361.  
  5362. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  5363. a:850)
  5364.  
  5365. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  5366. etService.java:153)
  5367.  
  5368. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  5369. vice.java:81)
  5370.  
  5371. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  5372. Helper.java:74)
  5373.  
  5374. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5375. uteUpdateRequestOnce(TransportShardBulkAction.java:343)
  5376.  
  5377. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5378. uteUpdateRequest(TransportShardBulkAction.java:406)
  5379.  
  5380. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5381. uteBulkItemRequest(TransportShardBulkAction.java:240)
  5382.  
  5383. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  5384. ormOnPrimary(TransportShardBulkAction.java:123)
  5385.  
  5386. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5387. dOperationOnPrimary(TransportShardBulkAction.java:110)
  5388.  
  5389. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5390. dOperationOnPrimary(TransportShardBulkAction.java:72)
  5391.  
  5392. at org.elasticsearch.action.support.replication.TransportRepli\
  5393. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5394. java:1034)
  5395.  
  5396. at org.elasticsearch.action.support.replication.TransportRepli\
  5397. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5398. java:1012)
  5399.  
  5400. at org.elasticsearch.action.support.replication.ReplicationOpe\
  5401. ration.execute(ReplicationOperation.java:103)
  5402.  
  5403. at org.elasticsearch.action.support.replication.TransportRepli\
  5404. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5405. java:359)
  5406.  
  5407. at org.elasticsearch.action.support.replication.TransportRepli\
  5408. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5409. java:299)
  5410.  
  5411. at org.elasticsearch.action.support.replication.TransportRepli\
  5412. cationAction$1.onResponse(TransportReplicationAction.java:975)
  5413.  
  5414. at org.elasticsearch.action.support.replication.TransportRepli\
  5415. cationAction$1.onResponse(TransportReplicationAction.java:972)
  5416.  
  5417. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  5418. quire(IndexShardOperationPermits.java:238)
  5419.  
  5420. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  5421. ationPermit(IndexShard.java:2220)
  5422.  
  5423. at org.elasticsearch.action.support.replication.TransportRepli\
  5424. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  5425. ava:984)
  5426.  
  5427. at org.elasticsearch.action.support.replication.TransportRepli\
  5428. cationAction.access$500(TransportReplicationAction.java:98)
  5429.  
  5430. at org.elasticsearch.action.support.replication.TransportRepli\
  5431. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  5432. 320)
  5433.  
  5434. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5435. un(AbstractRunnable.java:37)
  5436.  
  5437. at org.elasticsearch.action.support.replication.TransportRepli\
  5438. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5439. tReplicationAction.java:295)
  5440.  
  5441. at org.elasticsearch.action.support.replication.TransportRepli\
  5442. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5443. tReplicationAction.java:282)
  5444.  
  5445. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  5446. essageReceived(RequestHandlerRegistry.java:66)
  5447.  
  5448. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  5449. ortService.java:656)
  5450.  
  5451. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  5452. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  5453.  
  5454. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5455. un(AbstractRunnable.java:37)
  5456.  
  5457. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  5458. lExecutor.java:1149)
  5459.  
  5460. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  5461. olExecutor.java:624)
  5462.  
  5463. at java.lang.Thread.run(Thread.java:748)
  5464.  
  5465.  
  5466.  
  5467. "elasticsearch[fmfe16][bulk][T#9]" "
  5468. #388 "
  5469. daemon "
  5470. prio=5 "
  5471. os_prio=15 "
  5472. tid=0x0000000ae9abf000 "
  5473. nid=0x18eac "
  5474. waiting on condition "
  5475. [0x00007fffc7473000]
  5476.  
  5477. java.lang.Thread.State: WAITING (parking)
  5478.  
  5479. at sun.misc.Unsafe.park(Native Method)
  5480.  
  5481. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  5482. rrent.locks.ReentrantLock$NonfairSync)
  5483.  
  5484. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  5485. a:175)
  5486.  
  5487. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  5488. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  5489.  
  5490. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5491. reQueued(AbstractQueuedSynchronizer.java:870)
  5492.  
  5493. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5494. re(AbstractQueuedSynchronizer.java:1199)
  5495.  
  5496. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  5497. eentrantLock.java:209)
  5498.  
  5499. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  5500. .java:285)
  5501.  
  5502. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  5503. KeyedLock.java:76)
  5504.  
  5505. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  5506. iveVersionMap.java:431)
  5507.  
  5508. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  5509. ombstones(InternalEngine.java:1661)
  5510.  
  5511. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  5512. etedTombstones(InternalEngine.java:1348)
  5513.  
  5514. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  5515. nalEngine.java:1427)
  5516.  
  5517. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  5518. ngine.java:637)
  5519.  
  5520. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  5521. a:850)
  5522.  
  5523. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  5524. etService.java:153)
  5525.  
  5526. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  5527. vice.java:81)
  5528.  
  5529. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  5530. Helper.java:74)
  5531.  
  5532. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5533. uteUpdateRequestOnce(TransportShardBulkAction.java:343)
  5534.  
  5535. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5536. uteUpdateRequest(TransportShardBulkAction.java:406)
  5537.  
  5538. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5539. uteBulkItemRequest(TransportShardBulkAction.java:240)
  5540.  
  5541. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  5542. ormOnPrimary(TransportShardBulkAction.java:123)
  5543.  
  5544. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5545. dOperationOnPrimary(TransportShardBulkAction.java:110)
  5546.  
  5547. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5548. dOperationOnPrimary(TransportShardBulkAction.java:72)
  5549.  
  5550. at org.elasticsearch.action.support.replication.TransportRepli\
  5551. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5552. java:1034)
  5553.  
  5554. at org.elasticsearch.action.support.replication.TransportRepli\
  5555. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5556. java:1012)
  5557.  
  5558. at org.elasticsearch.action.support.replication.ReplicationOpe\
  5559. ration.execute(ReplicationOperation.java:103)
  5560.  
  5561. at org.elasticsearch.action.support.replication.TransportRepli\
  5562. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5563. java:359)
  5564.  
  5565. at org.elasticsearch.action.support.replication.TransportRepli\
  5566. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5567. java:299)
  5568.  
  5569. at org.elasticsearch.action.support.replication.TransportRepli\
  5570. cationAction$1.onResponse(TransportReplicationAction.java:975)
  5571.  
  5572. at org.elasticsearch.action.support.replication.TransportRepli\
  5573. cationAction$1.onResponse(TransportReplicationAction.java:972)
  5574.  
  5575. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  5576. quire(IndexShardOperationPermits.java:238)
  5577.  
  5578. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  5579. ationPermit(IndexShard.java:2220)
  5580.  
  5581. at org.elasticsearch.action.support.replication.TransportRepli\
  5582. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  5583. ava:984)
  5584.  
  5585. at org.elasticsearch.action.support.replication.TransportRepli\
  5586. cationAction.access$500(TransportReplicationAction.java:98)
  5587.  
  5588. at org.elasticsearch.action.support.replication.TransportRepli\
  5589. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  5590. 320)
  5591.  
  5592. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5593. un(AbstractRunnable.java:37)
  5594.  
  5595. at org.elasticsearch.action.support.replication.TransportRepli\
  5596. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5597. tReplicationAction.java:295)
  5598.  
  5599. at org.elasticsearch.action.support.replication.TransportRepli\
  5600. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5601. tReplicationAction.java:282)
  5602.  
  5603. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  5604. essageReceived(RequestHandlerRegistry.java:66)
  5605.  
  5606. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  5607. ortService.java:656)
  5608.  
  5609. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  5610. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  5611.  
  5612. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5613. un(AbstractRunnable.java:37)
  5614.  
  5615. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  5616. lExecutor.java:1149)
  5617.  
  5618. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  5619. olExecutor.java:624)
  5620.  
  5621. at java.lang.Thread.run(Thread.java:748)
  5622.  
  5623.  
  5624.  
  5625. "elasticsearch[fmfe16][bulk][T#8]" "
  5626. #383 "
  5627. daemon "
  5628. prio=5 "
  5629. os_prio=15 "
  5630. tid=0x00000008148b2800 "
  5631. nid=0x1895f "
  5632. waiting on condition "
  5633. [0x00007fffc6b6a000]
  5634.  
  5635. java.lang.Thread.State: WAITING (parking)
  5636.  
  5637. at sun.misc.Unsafe.park(Native Method)
  5638.  
  5639. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  5640. rrent.locks.ReentrantLock$NonfairSync)
  5641.  
  5642. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  5643. a:175)
  5644.  
  5645. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  5646. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  5647.  
  5648. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5649. reQueued(AbstractQueuedSynchronizer.java:870)
  5650.  
  5651. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5652. re(AbstractQueuedSynchronizer.java:1199)
  5653.  
  5654. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  5655. eentrantLock.java:209)
  5656.  
  5657. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  5658. .java:285)
  5659.  
  5660. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  5661. KeyedLock.java:76)
  5662.  
  5663. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  5664. iveVersionMap.java:431)
  5665.  
  5666. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  5667. ombstones(InternalEngine.java:1661)
  5668.  
  5669. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  5670. etedTombstones(InternalEngine.java:1348)
  5671.  
  5672. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  5673. nalEngine.java:1427)
  5674.  
  5675. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  5676. ngine.java:637)
  5677.  
  5678. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  5679. a:850)
  5680.  
  5681. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  5682. etService.java:153)
  5683.  
  5684. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  5685. vice.java:81)
  5686.  
  5687. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  5688. Helper.java:74)
  5689.  
  5690. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5691. uteUpdateRequestOnce(TransportShardBulkAction.java:343)
  5692.  
  5693. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5694. uteUpdateRequest(TransportShardBulkAction.java:406)
  5695.  
  5696. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5697. uteBulkItemRequest(TransportShardBulkAction.java:240)
  5698.  
  5699. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  5700. ormOnPrimary(TransportShardBulkAction.java:123)
  5701.  
  5702. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5703. dOperationOnPrimary(TransportShardBulkAction.java:110)
  5704.  
  5705. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5706. dOperationOnPrimary(TransportShardBulkAction.java:72)
  5707.  
  5708. at org.elasticsearch.action.support.replication.TransportRepli\
  5709. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5710. java:1034)
  5711.  
  5712. at org.elasticsearch.action.support.replication.TransportRepli\
  5713. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5714. java:1012)
  5715.  
  5716. at org.elasticsearch.action.support.replication.ReplicationOpe\
  5717. ration.execute(ReplicationOperation.java:103)
  5718.  
  5719. at org.elasticsearch.action.support.replication.TransportRepli\
  5720. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5721. java:359)
  5722.  
  5723. at org.elasticsearch.action.support.replication.TransportRepli\
  5724. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5725. java:299)
  5726.  
  5727. at org.elasticsearch.action.support.replication.TransportRepli\
  5728. cationAction$1.onResponse(TransportReplicationAction.java:975)
  5729.  
  5730. at org.elasticsearch.action.support.replication.TransportRepli\
  5731. cationAction$1.onResponse(TransportReplicationAction.java:972)
  5732.  
  5733. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  5734. quire(IndexShardOperationPermits.java:238)
  5735.  
  5736. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  5737. ationPermit(IndexShard.java:2220)
  5738.  
  5739. at org.elasticsearch.action.support.replication.TransportRepli\
  5740. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  5741. ava:984)
  5742.  
  5743. at org.elasticsearch.action.support.replication.TransportRepli\
  5744. cationAction.access$500(TransportReplicationAction.java:98)
  5745.  
  5746. at org.elasticsearch.action.support.replication.TransportRepli\
  5747. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  5748. 320)
  5749.  
  5750. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5751. un(AbstractRunnable.java:37)
  5752.  
  5753. at org.elasticsearch.action.support.replication.TransportRepli\
  5754. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5755. tReplicationAction.java:295)
  5756.  
  5757. at org.elasticsearch.action.support.replication.TransportRepli\
  5758. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  5759. tReplicationAction.java:282)
  5760.  
  5761. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  5762. essageReceived(RequestHandlerRegistry.java:66)
  5763.  
  5764. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  5765. ortService.java:656)
  5766.  
  5767. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  5768. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  5769.  
  5770. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5771. un(AbstractRunnable.java:37)
  5772.  
  5773. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  5774. lExecutor.java:1149)
  5775.  
  5776. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  5777. olExecutor.java:624)
  5778.  
  5779. at java.lang.Thread.run(Thread.java:748)
  5780.  
  5781.  
  5782.  
  5783. "elasticsearch[fmfe16][index][T#3]" "
  5784. #382 "
  5785. daemon "
  5786. prio=5 "
  5787. os_prio=15 "
  5788. tid=0x0000000b0d0ae000 "
  5789. nid=0x1895d "
  5790. waiting on condition "
  5791. [0x00007fffc6c6b000]
  5792.  
  5793. java.lang.Thread.State: WAITING (parking)
  5794.  
  5795. at sun.misc.Unsafe.park(Native Method)
  5796.  
  5797. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  5798. rrent.locks.ReentrantLock$NonfairSync)
  5799.  
  5800. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  5801. a:175)
  5802.  
  5803. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  5804. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  5805.  
  5806. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5807. reQueued(AbstractQueuedSynchronizer.java:870)
  5808.  
  5809. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5810. re(AbstractQueuedSynchronizer.java:1199)
  5811.  
  5812. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  5813. eentrantLock.java:209)
  5814.  
  5815. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  5816. .java:285)
  5817.  
  5818. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  5819. KeyedLock.java:76)
  5820.  
  5821. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  5822. iveVersionMap.java:431)
  5823.  
  5824. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  5825. ombstones(InternalEngine.java:1661)
  5826.  
  5827. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  5828. etedTombstones(InternalEngine.java:1348)
  5829.  
  5830. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  5831. nalEngine.java:1427)
  5832.  
  5833. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  5834. ngine.java:637)
  5835.  
  5836. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  5837. a:850)
  5838.  
  5839. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  5840. etService.java:153)
  5841.  
  5842. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  5843. vice.java:81)
  5844.  
  5845. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  5846. Helper.java:74)
  5847.  
  5848. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  5849. Operation(TransportUpdateAction.java:174)
  5850.  
  5851. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  5852. Operation(TransportUpdateAction.java:167)
  5853.  
  5854. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  5855. Operation(TransportUpdateAction.java:67)
  5856.  
  5857. at org.elasticsearch.action.support.single.instance.TransportI\
  5858. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  5859. nsportInstanceSingleOperationAction.java:243)
  5860.  
  5861. at org.elasticsearch.action.support.single.instance.TransportI\
  5862. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  5863. nsportInstanceSingleOperationAction.java:239)
  5864.  
  5865. at org.elasticsearch.transport.TransportRequestHandler.message\
  5866. Received(TransportRequestHandler.java:30)
  5867.  
  5868. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  5869. essageReceived(RequestHandlerRegistry.java:66)
  5870.  
  5871. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  5872. un(TcpTransport.java:1555)
  5873.  
  5874. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  5875. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  5876.  
  5877. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  5878. un(AbstractRunnable.java:37)
  5879.  
  5880. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  5881. lExecutor.java:1149)
  5882.  
  5883. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  5884. olExecutor.java:624)
  5885.  
  5886. at java.lang.Thread.run(Thread.java:748)
  5887.  
  5888.  
  5889.  
  5890. "elasticsearch[fmfe16][bulk][T#7]" "
  5891. #381 "
  5892. daemon "
  5893. prio=5 "
  5894. os_prio=15 "
  5895. tid=0x0000000b0c66e800 "
  5896. nid=0x18952 "
  5897. waiting on condition "
  5898. [0x00007fffc6d6c000]
  5899.  
  5900. java.lang.Thread.State: WAITING (parking)
  5901.  
  5902. at sun.misc.Unsafe.park(Native Method)
  5903.  
  5904. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  5905. rrent.locks.ReentrantLock$NonfairSync)
  5906.  
  5907. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  5908. a:175)
  5909.  
  5910. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  5911. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  5912.  
  5913. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5914. reQueued(AbstractQueuedSynchronizer.java:870)
  5915.  
  5916. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  5917. re(AbstractQueuedSynchronizer.java:1199)
  5918.  
  5919. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  5920. eentrantLock.java:209)
  5921.  
  5922. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  5923. .java:285)
  5924.  
  5925. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  5926. KeyedLock.java:76)
  5927.  
  5928. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  5929. iveVersionMap.java:431)
  5930.  
  5931. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  5932. ombstones(InternalEngine.java:1661)
  5933.  
  5934. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  5935. etedTombstones(InternalEngine.java:1348)
  5936.  
  5937. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  5938. alEngine.java:1211)
  5939.  
  5940. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  5941. java:839)
  5942.  
  5943. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  5944. on(IndexShard.java:806)
  5945.  
  5946. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  5947. onOnPrimary(IndexShard.java:770)
  5948.  
  5949. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5950. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  5951.  
  5952. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5953. uteDeleteRequest(TransportShardBulkAction.java:154)
  5954.  
  5955. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  5956. uteBulkItemRequest(TransportShardBulkAction.java:244)
  5957.  
  5958. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  5959. ormOnPrimary(TransportShardBulkAction.java:123)
  5960.  
  5961. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5962. dOperationOnPrimary(TransportShardBulkAction.java:110)
  5963.  
  5964. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  5965. dOperationOnPrimary(TransportShardBulkAction.java:72)
  5966.  
  5967. at org.elasticsearch.action.support.replication.TransportRepli\
  5968. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5969. java:1034)
  5970.  
  5971. at org.elasticsearch.action.support.replication.TransportRepli\
  5972. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  5973. java:1012)
  5974.  
  5975. at org.elasticsearch.action.support.replication.ReplicationOpe\
  5976. ration.execute(ReplicationOperation.java:103)
  5977.  
  5978. at org.elasticsearch.action.support.replication.TransportRepli\
  5979. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5980. java:359)
  5981.  
  5982. at org.elasticsearch.action.support.replication.TransportRepli\
  5983. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  5984. java:299)
  5985.  
  5986. at org.elasticsearch.action.support.replication.TransportRepli\
  5987. cationAction$1.onResponse(TransportReplicationAction.java:975)
  5988.  
  5989. at org.elasticsearch.action.support.replication.TransportRepli\
  5990. cationAction$1.onResponse(TransportReplicationAction.java:972)
  5991.  
  5992. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  5993. quire(IndexShardOperationPermits.java:238)
  5994.  
  5995. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  5996. ationPermit(IndexShard.java:2220)
  5997.  
  5998. at org.elasticsearch.action.support.replication.TransportRepli\
  5999. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  6000. ava:984)
  6001.  
  6002. at org.elasticsearch.action.support.replication.TransportRepli\
  6003. cationAction.access$500(TransportReplicationAction.java:98)
  6004.  
  6005. at org.elasticsearch.action.support.replication.TransportRepli\
  6006. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  6007. 320)
  6008.  
  6009. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6010. un(AbstractRunnable.java:37)
  6011.  
  6012. at org.elasticsearch.action.support.replication.TransportRepli\
  6013. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6014. tReplicationAction.java:295)
  6015.  
  6016. at org.elasticsearch.action.support.replication.TransportRepli\
  6017. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6018. tReplicationAction.java:282)
  6019.  
  6020. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  6021. essageReceived(RequestHandlerRegistry.java:66)
  6022.  
  6023. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  6024. ortService.java:656)
  6025.  
  6026. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  6027. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  6028.  
  6029. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6030. un(AbstractRunnable.java:37)
  6031.  
  6032. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  6033. lExecutor.java:1149)
  6034.  
  6035. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  6036. olExecutor.java:624)
  6037.  
  6038. at java.lang.Thread.run(Thread.java:748)
  6039.  
  6040.  
  6041.  
  6042. "elasticsearch[fmfe16][bulk][T#6]" "
  6043. #380 "
  6044. daemon "
  6045. prio=5 "
  6046. os_prio=15 "
  6047. tid=0x00000008142b0800 "
  6048. nid=0x188f4 "
  6049. waiting for monitor entry "
  6050. [0x00007fffc6e6d000]
  6051.  
  6052. java.lang.Thread.State: BLOCKED (on object monitor)
  6053.  
  6054. at org.elasticsearch.index.engine.InternalEngine.getVersionFro\
  6055. mMap(InternalEngine.java:732)
  6056.  
  6057. - waiting to lock <0x00000008b40309c8> "
  6058. (a org.elasticsearch.index.engine.LiveVersionMap)
  6059.  
  6060. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  6061. ngine.java:627)
  6062.  
  6063. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  6064. a:850)
  6065.  
  6066. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  6067. etService.java:153)
  6068.  
  6069. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  6070. vice.java:81)
  6071.  
  6072. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  6073. Helper.java:74)
  6074.  
  6075. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6076. uteUpdateRequestOnce(TransportShardBulkAction.java:343)
  6077.  
  6078. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6079. uteUpdateRequest(TransportShardBulkAction.java:406)
  6080.  
  6081. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6082. uteBulkItemRequest(TransportShardBulkAction.java:240)
  6083.  
  6084. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  6085. ormOnPrimary(TransportShardBulkAction.java:123)
  6086.  
  6087. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6088. dOperationOnPrimary(TransportShardBulkAction.java:110)
  6089.  
  6090. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6091. dOperationOnPrimary(TransportShardBulkAction.java:72)
  6092.  
  6093. at org.elasticsearch.action.support.replication.TransportRepli\
  6094. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6095. java:1034)
  6096.  
  6097. at org.elasticsearch.action.support.replication.TransportRepli\
  6098. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6099. java:1012)
  6100.  
  6101. at org.elasticsearch.action.support.replication.ReplicationOpe\
  6102. ration.execute(ReplicationOperation.java:103)
  6103.  
  6104. at org.elasticsearch.action.support.replication.TransportRepli\
  6105. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6106. java:359)
  6107.  
  6108. at org.elasticsearch.action.support.replication.TransportRepli\
  6109. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6110. java:299)
  6111.  
  6112. at org.elasticsearch.action.support.replication.TransportRepli\
  6113. cationAction$1.onResponse(TransportReplicationAction.java:975)
  6114.  
  6115. at org.elasticsearch.action.support.replication.TransportRepli\
  6116. cationAction$1.onResponse(TransportReplicationAction.java:972)
  6117.  
  6118. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  6119. quire(IndexShardOperationPermits.java:238)
  6120.  
  6121. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  6122. ationPermit(IndexShard.java:2220)
  6123.  
  6124. at org.elasticsearch.action.support.replication.TransportRepli\
  6125. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  6126. ava:984)
  6127.  
  6128. at org.elasticsearch.action.support.replication.TransportRepli\
  6129. cationAction.access$500(TransportReplicationAction.java:98)
  6130.  
  6131. at org.elasticsearch.action.support.replication.TransportRepli\
  6132. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  6133. 320)
  6134.  
  6135. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6136. un(AbstractRunnable.java:37)
  6137.  
  6138. at org.elasticsearch.action.support.replication.TransportRepli\
  6139. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6140. tReplicationAction.java:295)
  6141.  
  6142. at org.elasticsearch.action.support.replication.TransportRepli\
  6143. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6144. tReplicationAction.java:282)
  6145.  
  6146. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  6147. essageReceived(RequestHandlerRegistry.java:66)
  6148.  
  6149. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  6150. ortService.java:656)
  6151.  
  6152. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  6153. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  6154.  
  6155. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6156. un(AbstractRunnable.java:37)
  6157.  
  6158. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  6159. lExecutor.java:1149)
  6160.  
  6161. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  6162. olExecutor.java:624)
  6163.  
  6164. at java.lang.Thread.run(Thread.java:748)
  6165.  
  6166.  
  6167.  
  6168. "elasticsearch[fmfe16][index][T#2]" "
  6169. #379 "
  6170. daemon "
  6171. prio=5 "
  6172. os_prio=15 "
  6173. tid=0x0000000ae8ccf000 "
  6174. nid=0x188e4 "
  6175. waiting on condition "
  6176. [0x00007fffc6f6e000]
  6177.  
  6178. java.lang.Thread.State: WAITING (parking)
  6179.  
  6180. at sun.misc.Unsafe.park(Native Method)
  6181.  
  6182. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  6183. rrent.locks.ReentrantLock$NonfairSync)
  6184.  
  6185. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  6186. a:175)
  6187.  
  6188. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  6189. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  6190.  
  6191. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6192. reQueued(AbstractQueuedSynchronizer.java:870)
  6193.  
  6194. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6195. re(AbstractQueuedSynchronizer.java:1199)
  6196.  
  6197. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  6198. eentrantLock.java:209)
  6199.  
  6200. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  6201. .java:285)
  6202.  
  6203. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  6204. KeyedLock.java:76)
  6205.  
  6206. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  6207. iveVersionMap.java:431)
  6208.  
  6209. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  6210. ombstones(InternalEngine.java:1661)
  6211.  
  6212. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  6213. etedTombstones(InternalEngine.java:1348)
  6214.  
  6215. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  6216. nalEngine.java:1427)
  6217.  
  6218. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  6219. ngine.java:637)
  6220.  
  6221. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  6222. a:850)
  6223.  
  6224. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  6225. etService.java:153)
  6226.  
  6227. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  6228. vice.java:81)
  6229.  
  6230. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  6231. Helper.java:74)
  6232.  
  6233. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  6234. Operation(TransportUpdateAction.java:174)
  6235.  
  6236. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  6237. Operation(TransportUpdateAction.java:167)
  6238.  
  6239. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  6240. Operation(TransportUpdateAction.java:67)
  6241.  
  6242. at org.elasticsearch.action.support.single.instance.TransportI\
  6243. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  6244. nsportInstanceSingleOperationAction.java:243)
  6245.  
  6246. at org.elasticsearch.action.support.single.instance.TransportI\
  6247. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  6248. nsportInstanceSingleOperationAction.java:239)
  6249.  
  6250. at org.elasticsearch.transport.TransportRequestHandler.message\
  6251. Received(TransportRequestHandler.java:30)
  6252.  
  6253. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  6254. essageReceived(RequestHandlerRegistry.java:66)
  6255.  
  6256. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  6257. un(TcpTransport.java:1555)
  6258.  
  6259. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  6260. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  6261.  
  6262. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6263. un(AbstractRunnable.java:37)
  6264.  
  6265. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  6266. lExecutor.java:1149)
  6267.  
  6268. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  6269. olExecutor.java:624)
  6270.  
  6271. at java.lang.Thread.run(Thread.java:748)
  6272.  
  6273.  
  6274.  
  6275. "elasticsearch[fmfe16][bulk][T#5]" "
  6276. #378 "
  6277. daemon "
  6278. prio=5 "
  6279. os_prio=15 "
  6280. tid=0x0000000b04660800 "
  6281. nid=0x188e2 "
  6282. waiting on condition "
  6283. [0x00007fffc706f000]
  6284.  
  6285. java.lang.Thread.State: WAITING (parking)
  6286.  
  6287. at sun.misc.Unsafe.park(Native Method)
  6288.  
  6289. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  6290. rrent.locks.ReentrantLock$NonfairSync)
  6291.  
  6292. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  6293. a:175)
  6294.  
  6295. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  6296. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  6297.  
  6298. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6299. reQueued(AbstractQueuedSynchronizer.java:870)
  6300.  
  6301. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6302. re(AbstractQueuedSynchronizer.java:1199)
  6303.  
  6304. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  6305. eentrantLock.java:209)
  6306.  
  6307. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  6308. .java:285)
  6309.  
  6310. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  6311. KeyedLock.java:76)
  6312.  
  6313. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  6314. iveVersionMap.java:431)
  6315.  
  6316. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  6317. ombstones(InternalEngine.java:1661)
  6318.  
  6319. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  6320. etedTombstones(InternalEngine.java:1348)
  6321.  
  6322. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  6323. alEngine.java:1211)
  6324.  
  6325. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  6326. java:839)
  6327.  
  6328. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  6329. on(IndexShard.java:806)
  6330.  
  6331. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  6332. onOnPrimary(IndexShard.java:770)
  6333.  
  6334. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6335. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  6336.  
  6337. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6338. uteDeleteRequest(TransportShardBulkAction.java:154)
  6339.  
  6340. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6341. uteBulkItemRequest(TransportShardBulkAction.java:244)
  6342.  
  6343. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  6344. ormOnPrimary(TransportShardBulkAction.java:123)
  6345.  
  6346. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6347. dOperationOnPrimary(TransportShardBulkAction.java:110)
  6348.  
  6349. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6350. dOperationOnPrimary(TransportShardBulkAction.java:72)
  6351.  
  6352. at org.elasticsearch.action.support.replication.TransportRepli\
  6353. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6354. java:1034)
  6355.  
  6356. at org.elasticsearch.action.support.replication.TransportRepli\
  6357. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6358. java:1012)
  6359.  
  6360. at org.elasticsearch.action.support.replication.ReplicationOpe\
  6361. ration.execute(ReplicationOperation.java:103)
  6362.  
  6363. at org.elasticsearch.action.support.replication.TransportRepli\
  6364. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6365. java:359)
  6366.  
  6367. at org.elasticsearch.action.support.replication.TransportRepli\
  6368. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6369. java:299)
  6370.  
  6371. at org.elasticsearch.action.support.replication.TransportRepli\
  6372. cationAction$1.onResponse(TransportReplicationAction.java:975)
  6373.  
  6374. at org.elasticsearch.action.support.replication.TransportRepli\
  6375. cationAction$1.onResponse(TransportReplicationAction.java:972)
  6376.  
  6377. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  6378. quire(IndexShardOperationPermits.java:238)
  6379.  
  6380. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  6381. ationPermit(IndexShard.java:2220)
  6382.  
  6383. at org.elasticsearch.action.support.replication.TransportRepli\
  6384. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  6385. ava:984)
  6386.  
  6387. at org.elasticsearch.action.support.replication.TransportRepli\
  6388. cationAction.access$500(TransportReplicationAction.java:98)
  6389.  
  6390. at org.elasticsearch.action.support.replication.TransportRepli\
  6391. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  6392. 320)
  6393.  
  6394. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6395. un(AbstractRunnable.java:37)
  6396.  
  6397. at org.elasticsearch.action.support.replication.TransportRepli\
  6398. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6399. tReplicationAction.java:295)
  6400.  
  6401. at org.elasticsearch.action.support.replication.TransportRepli\
  6402. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6403. tReplicationAction.java:282)
  6404.  
  6405. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  6406. essageReceived(RequestHandlerRegistry.java:66)
  6407.  
  6408. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  6409. ortService.java:656)
  6410.  
  6411. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  6412. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  6413.  
  6414. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6415. un(AbstractRunnable.java:37)
  6416.  
  6417. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  6418. lExecutor.java:1149)
  6419.  
  6420. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  6421. olExecutor.java:624)
  6422.  
  6423. at java.lang.Thread.run(Thread.java:748)
  6424.  
  6425.  
  6426.  
  6427. "elasticsearch[fmfe16][index][T#1]" "
  6428. #377 "
  6429. daemon "
  6430. prio=5 "
  6431. os_prio=15 "
  6432. tid=0x0000000b0d0ad800 "
  6433. nid=0x188de "
  6434. waiting on condition "
  6435. [0x00007fffc7170000]
  6436.  
  6437. java.lang.Thread.State: WAITING (parking)
  6438.  
  6439. at sun.misc.Unsafe.park(Native Method)
  6440.  
  6441. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  6442. rrent.locks.ReentrantLock$NonfairSync)
  6443.  
  6444. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  6445. a:175)
  6446.  
  6447. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  6448. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  6449.  
  6450. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6451. reQueued(AbstractQueuedSynchronizer.java:870)
  6452.  
  6453. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6454. re(AbstractQueuedSynchronizer.java:1199)
  6455.  
  6456. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  6457. eentrantLock.java:209)
  6458.  
  6459. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  6460. .java:285)
  6461.  
  6462. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  6463. KeyedLock.java:76)
  6464.  
  6465. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  6466. iveVersionMap.java:431)
  6467.  
  6468. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  6469. ombstones(InternalEngine.java:1661)
  6470.  
  6471. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  6472. etedTombstones(InternalEngine.java:1348)
  6473.  
  6474. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  6475. nalEngine.java:1427)
  6476.  
  6477. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  6478. ngine.java:637)
  6479.  
  6480. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  6481. a:850)
  6482.  
  6483. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  6484. etService.java:153)
  6485.  
  6486. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  6487. vice.java:81)
  6488.  
  6489. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  6490. Helper.java:74)
  6491.  
  6492. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  6493. Operation(TransportUpdateAction.java:174)
  6494.  
  6495. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  6496. Operation(TransportUpdateAction.java:167)
  6497.  
  6498. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  6499. Operation(TransportUpdateAction.java:67)
  6500.  
  6501. at org.elasticsearch.action.support.single.instance.TransportI\
  6502. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  6503. nsportInstanceSingleOperationAction.java:243)
  6504.  
  6505. at org.elasticsearch.action.support.single.instance.TransportI\
  6506. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  6507. nsportInstanceSingleOperationAction.java:239)
  6508.  
  6509. at org.elasticsearch.transport.TransportRequestHandler.message\
  6510. Received(TransportRequestHandler.java:30)
  6511.  
  6512. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  6513. essageReceived(RequestHandlerRegistry.java:66)
  6514.  
  6515. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  6516. un(TcpTransport.java:1555)
  6517.  
  6518. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  6519. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  6520.  
  6521. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6522. un(AbstractRunnable.java:37)
  6523.  
  6524. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  6525. lExecutor.java:1149)
  6526.  
  6527. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  6528. olExecutor.java:624)
  6529.  
  6530. at java.lang.Thread.run(Thread.java:748)
  6531.  
  6532.  
  6533.  
  6534. "elasticsearch[fmfe16][bulk][T#4]" "
  6535. #376 "
  6536. daemon "
  6537. prio=5 "
  6538. os_prio=15 "
  6539. tid=0x0000000ae9883800 "
  6540. nid=0x1879f "
  6541. waiting on condition "
  6542. [0x00007fffc7271000]
  6543.  
  6544. java.lang.Thread.State: WAITING (parking)
  6545.  
  6546. at sun.misc.Unsafe.park(Native Method)
  6547.  
  6548. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  6549. rrent.locks.ReentrantLock$NonfairSync)
  6550.  
  6551. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  6552. a:175)
  6553.  
  6554. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  6555. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  6556.  
  6557. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6558. reQueued(AbstractQueuedSynchronizer.java:870)
  6559.  
  6560. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6561. re(AbstractQueuedSynchronizer.java:1199)
  6562.  
  6563. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  6564. eentrantLock.java:209)
  6565.  
  6566. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  6567. .java:285)
  6568.  
  6569. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  6570. KeyedLock.java:76)
  6571.  
  6572. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  6573. iveVersionMap.java:431)
  6574.  
  6575. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  6576. ombstones(InternalEngine.java:1661)
  6577.  
  6578. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  6579. etedTombstones(InternalEngine.java:1348)
  6580.  
  6581. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  6582. alEngine.java:1211)
  6583.  
  6584. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  6585. java:839)
  6586.  
  6587. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  6588. on(IndexShard.java:806)
  6589.  
  6590. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  6591. onOnPrimary(IndexShard.java:770)
  6592.  
  6593. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6594. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  6595.  
  6596. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6597. uteDeleteRequest(TransportShardBulkAction.java:154)
  6598.  
  6599. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6600. uteBulkItemRequest(TransportShardBulkAction.java:244)
  6601.  
  6602. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  6603. ormOnPrimary(TransportShardBulkAction.java:123)
  6604.  
  6605. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6606. dOperationOnPrimary(TransportShardBulkAction.java:110)
  6607.  
  6608. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6609. dOperationOnPrimary(TransportShardBulkAction.java:72)
  6610.  
  6611. at org.elasticsearch.action.support.replication.TransportRepli\
  6612. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6613. java:1034)
  6614.  
  6615. at org.elasticsearch.action.support.replication.TransportRepli\
  6616. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6617. java:1012)
  6618.  
  6619. at org.elasticsearch.action.support.replication.ReplicationOpe\
  6620. ration.execute(ReplicationOperation.java:103)
  6621.  
  6622. at org.elasticsearch.action.support.replication.TransportRepli\
  6623. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6624. java:359)
  6625.  
  6626. at org.elasticsearch.action.support.replication.TransportRepli\
  6627. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6628. java:299)
  6629.  
  6630. at org.elasticsearch.action.support.replication.TransportRepli\
  6631. cationAction$1.onResponse(TransportReplicationAction.java:975)
  6632.  
  6633. at org.elasticsearch.action.support.replication.TransportRepli\
  6634. cationAction$1.onResponse(TransportReplicationAction.java:972)
  6635.  
  6636. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  6637. quire(IndexShardOperationPermits.java:238)
  6638.  
  6639. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  6640. ationPermit(IndexShard.java:2220)
  6641.  
  6642. at org.elasticsearch.action.support.replication.TransportRepli\
  6643. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  6644. ava:984)
  6645.  
  6646. at org.elasticsearch.action.support.replication.TransportRepli\
  6647. cationAction.access$500(TransportReplicationAction.java:98)
  6648.  
  6649. at org.elasticsearch.action.support.replication.TransportRepli\
  6650. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  6651. 320)
  6652.  
  6653. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6654. un(AbstractRunnable.java:37)
  6655.  
  6656. at org.elasticsearch.action.support.replication.TransportRepli\
  6657. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6658. tReplicationAction.java:295)
  6659.  
  6660. at org.elasticsearch.action.support.replication.TransportRepli\
  6661. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6662. tReplicationAction.java:282)
  6663.  
  6664. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  6665. essageReceived(RequestHandlerRegistry.java:66)
  6666.  
  6667. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  6668. ortService.java:656)
  6669.  
  6670. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  6671. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  6672.  
  6673. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6674. un(AbstractRunnable.java:37)
  6675.  
  6676. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  6677. lExecutor.java:1149)
  6678.  
  6679. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  6680. olExecutor.java:624)
  6681.  
  6682. at java.lang.Thread.run(Thread.java:748)
  6683.  
  6684.  
  6685.  
  6686. "elasticsearch[fmfe16][bulk][T#3]" "
  6687. #373 "
  6688. daemon "
  6689. prio=5 "
  6690. os_prio=15 "
  6691. tid=0x0000000ae9abe000 "
  6692. nid=0x188c7 "
  6693. waiting on condition "
  6694. [0x00007fffc7574000]
  6695.  
  6696. java.lang.Thread.State: WAITING (parking)
  6697.  
  6698. at sun.misc.Unsafe.park(Native Method)
  6699.  
  6700. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  6701. rrent.locks.ReentrantLock$NonfairSync)
  6702.  
  6703. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  6704. a:175)
  6705.  
  6706. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  6707. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  6708.  
  6709. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6710. reQueued(AbstractQueuedSynchronizer.java:870)
  6711.  
  6712. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6713. re(AbstractQueuedSynchronizer.java:1199)
  6714.  
  6715. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  6716. eentrantLock.java:209)
  6717.  
  6718. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  6719. .java:285)
  6720.  
  6721. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  6722. KeyedLock.java:76)
  6723.  
  6724. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  6725. iveVersionMap.java:431)
  6726.  
  6727. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  6728. ombstones(InternalEngine.java:1661)
  6729.  
  6730. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  6731. etedTombstones(InternalEngine.java:1348)
  6732.  
  6733. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  6734. alEngine.java:1211)
  6735.  
  6736. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  6737. java:839)
  6738.  
  6739. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  6740. on(IndexShard.java:806)
  6741.  
  6742. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  6743. onOnPrimary(IndexShard.java:770)
  6744.  
  6745. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6746. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  6747.  
  6748. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6749. uteDeleteRequest(TransportShardBulkAction.java:154)
  6750.  
  6751. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6752. uteBulkItemRequest(TransportShardBulkAction.java:244)
  6753.  
  6754. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  6755. ormOnPrimary(TransportShardBulkAction.java:123)
  6756.  
  6757. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6758. dOperationOnPrimary(TransportShardBulkAction.java:110)
  6759.  
  6760. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6761. dOperationOnPrimary(TransportShardBulkAction.java:72)
  6762.  
  6763. at org.elasticsearch.action.support.replication.TransportRepli\
  6764. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6765. java:1034)
  6766.  
  6767. at org.elasticsearch.action.support.replication.TransportRepli\
  6768. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6769. java:1012)
  6770.  
  6771. at org.elasticsearch.action.support.replication.ReplicationOpe\
  6772. ration.execute(ReplicationOperation.java:103)
  6773.  
  6774. at org.elasticsearch.action.support.replication.TransportRepli\
  6775. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6776. java:359)
  6777.  
  6778. at org.elasticsearch.action.support.replication.TransportRepli\
  6779. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6780. java:299)
  6781.  
  6782. at org.elasticsearch.action.support.replication.TransportRepli\
  6783. cationAction$1.onResponse(TransportReplicationAction.java:975)
  6784.  
  6785. at org.elasticsearch.action.support.replication.TransportRepli\
  6786. cationAction$1.onResponse(TransportReplicationAction.java:972)
  6787.  
  6788. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  6789. quire(IndexShardOperationPermits.java:238)
  6790.  
  6791. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  6792. ationPermit(IndexShard.java:2220)
  6793.  
  6794. at org.elasticsearch.action.support.replication.TransportRepli\
  6795. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  6796. ava:984)
  6797.  
  6798. at org.elasticsearch.action.support.replication.TransportRepli\
  6799. cationAction.access$500(TransportReplicationAction.java:98)
  6800.  
  6801. at org.elasticsearch.action.support.replication.TransportRepli\
  6802. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  6803. 320)
  6804.  
  6805. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6806. un(AbstractRunnable.java:37)
  6807.  
  6808. at org.elasticsearch.action.support.replication.TransportRepli\
  6809. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6810. tReplicationAction.java:295)
  6811.  
  6812. at org.elasticsearch.action.support.replication.TransportRepli\
  6813. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6814. tReplicationAction.java:282)
  6815.  
  6816. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  6817. essageReceived(RequestHandlerRegistry.java:66)
  6818.  
  6819. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  6820. ortService.java:656)
  6821.  
  6822. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  6823. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  6824.  
  6825. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6826. un(AbstractRunnable.java:37)
  6827.  
  6828. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  6829. lExecutor.java:1149)
  6830.  
  6831. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  6832. olExecutor.java:624)
  6833.  
  6834. at java.lang.Thread.run(Thread.java:748)
  6835.  
  6836.  
  6837.  
  6838. "elasticsearch[fmfe16][bulk][T#2]" "
  6839. #372 "
  6840. daemon "
  6841. prio=5 "
  6842. os_prio=15 "
  6843. tid=0x0000000b010f4800 "
  6844. nid=0x18c11 "
  6845. waiting on condition "
  6846. [0x00007fffc7675000]
  6847.  
  6848. java.lang.Thread.State: WAITING (parking)
  6849.  
  6850. at sun.misc.Unsafe.park(Native Method)
  6851.  
  6852. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  6853. rrent.locks.ReentrantLock$NonfairSync)
  6854.  
  6855. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  6856. a:175)
  6857.  
  6858. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  6859. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  6860.  
  6861. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6862. reQueued(AbstractQueuedSynchronizer.java:870)
  6863.  
  6864. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  6865. re(AbstractQueuedSynchronizer.java:1199)
  6866.  
  6867. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  6868. eentrantLock.java:209)
  6869.  
  6870. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  6871. .java:285)
  6872.  
  6873. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  6874. KeyedLock.java:76)
  6875.  
  6876. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  6877. iveVersionMap.java:431)
  6878.  
  6879. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  6880. ombstones(InternalEngine.java:1661)
  6881.  
  6882. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  6883. etedTombstones(InternalEngine.java:1348)
  6884.  
  6885. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  6886. alEngine.java:1211)
  6887.  
  6888. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  6889. java:839)
  6890.  
  6891. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  6892. on(IndexShard.java:806)
  6893.  
  6894. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  6895. onOnPrimary(IndexShard.java:770)
  6896.  
  6897. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6898. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  6899.  
  6900. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6901. uteDeleteRequest(TransportShardBulkAction.java:154)
  6902.  
  6903. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  6904. uteBulkItemRequest(TransportShardBulkAction.java:244)
  6905.  
  6906. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  6907. ormOnPrimary(TransportShardBulkAction.java:123)
  6908.  
  6909. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6910. dOperationOnPrimary(TransportShardBulkAction.java:110)
  6911.  
  6912. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  6913. dOperationOnPrimary(TransportShardBulkAction.java:72)
  6914.  
  6915. at org.elasticsearch.action.support.replication.TransportRepli\
  6916. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6917. java:1034)
  6918.  
  6919. at org.elasticsearch.action.support.replication.TransportRepli\
  6920. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  6921. java:1012)
  6922.  
  6923. at org.elasticsearch.action.support.replication.ReplicationOpe\
  6924. ration.execute(ReplicationOperation.java:103)
  6925.  
  6926. at org.elasticsearch.action.support.replication.TransportRepli\
  6927. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6928. java:359)
  6929.  
  6930. at org.elasticsearch.action.support.replication.TransportRepli\
  6931. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  6932. java:299)
  6933.  
  6934. at org.elasticsearch.action.support.replication.TransportRepli\
  6935. cationAction$1.onResponse(TransportReplicationAction.java:975)
  6936.  
  6937. at org.elasticsearch.action.support.replication.TransportRepli\
  6938. cationAction$1.onResponse(TransportReplicationAction.java:972)
  6939.  
  6940. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  6941. quire(IndexShardOperationPermits.java:238)
  6942.  
  6943. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  6944. ationPermit(IndexShard.java:2220)
  6945.  
  6946. at org.elasticsearch.action.support.replication.TransportRepli\
  6947. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  6948. ava:984)
  6949.  
  6950. at org.elasticsearch.action.support.replication.TransportRepli\
  6951. cationAction.access$500(TransportReplicationAction.java:98)
  6952.  
  6953. at org.elasticsearch.action.support.replication.TransportRepli\
  6954. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  6955. 320)
  6956.  
  6957. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6958. un(AbstractRunnable.java:37)
  6959.  
  6960. at org.elasticsearch.action.support.replication.TransportRepli\
  6961. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6962. tReplicationAction.java:295)
  6963.  
  6964. at org.elasticsearch.action.support.replication.TransportRepli\
  6965. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  6966. tReplicationAction.java:282)
  6967.  
  6968. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  6969. essageReceived(RequestHandlerRegistry.java:66)
  6970.  
  6971. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  6972. ortService.java:656)
  6973.  
  6974. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  6975. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  6976.  
  6977. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  6978. un(AbstractRunnable.java:37)
  6979.  
  6980. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  6981. lExecutor.java:1149)
  6982.  
  6983. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  6984. olExecutor.java:624)
  6985.  
  6986. at java.lang.Thread.run(Thread.java:748)
  6987.  
  6988.  
  6989.  
  6990. "elasticsearch[fmfe16][generic][T#27]" "
  6991. #335 "
  6992. daemon "
  6993. prio=5 "
  6994. os_prio=15 "
  6995. tid=0x0000000b0465f800 "
  6996. nid=0x18c3a "
  6997. waiting on condition "
  6998. [0x00007fffc9b9a000]
  6999.  
  7000. java.lang.Thread.State: WAITING (parking)
  7001.  
  7002. at sun.misc.Unsafe.park(Native Method)
  7003.  
  7004. - parking to wait for <0x0000000872bdf1e0> (a org.elasticsear\
  7005. ch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
  7006.  
  7007. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7008. a:175)
  7009.  
  7010. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7011. ransferQueue.java:737)
  7012.  
  7013. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7014. rQueue.java:647)
  7015.  
  7016. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7017. rQueue.java:1269)
  7018.  
  7019. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7020. xecutor.java:1074)
  7021.  
  7022. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7023. lExecutor.java:1134)
  7024.  
  7025. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7026. olExecutor.java:624)
  7027.  
  7028. at java.lang.Thread.run(Thread.java:748)
  7029.  
  7030.  
  7031.  
  7032. "elasticsearch[fmfe16][flush][T#5]" "
  7033. #316 "
  7034. daemon "
  7035. prio=5 "
  7036. os_prio=15 "
  7037. tid=0x0000000b038e1000 "
  7038. nid=0x19a36 "
  7039. waiting on condition "
  7040. [0x00007fffcaead000]
  7041.  
  7042. java.lang.Thread.State: WAITING (parking)
  7043.  
  7044. at sun.misc.Unsafe.park(Native Method)
  7045.  
  7046. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  7047. rrent.locks.ReentrantLock$NonfairSync)
  7048.  
  7049. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7050. a:175)
  7051.  
  7052. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  7053. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  7054.  
  7055. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7056. reQueued(AbstractQueuedSynchronizer.java:870)
  7057.  
  7058. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7059. re(AbstractQueuedSynchronizer.java:1199)
  7060.  
  7061. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  7062. eentrantLock.java:209)
  7063.  
  7064. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  7065. .java:285)
  7066.  
  7067. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  7068. KeyedLock.java:76)
  7069.  
  7070. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  7071. iveVersionMap.java:431)
  7072.  
  7073. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  7074. ombstones(InternalEngine.java:1661)
  7075.  
  7076. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  7077. etedTombstones(InternalEngine.java:1348)
  7078.  
  7079. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  7080. nalEngine.java:1427)
  7081.  
  7082. at org.elasticsearch.index.engine.InternalEngine.flush(Interna\
  7083. lEngine.java:1565)
  7084.  
  7085. at org.elasticsearch.index.shard.IndexShard.flush(IndexShard.j\
  7086. ava:1030)
  7087.  
  7088. at org.elasticsearch.indices.flush.SyncedFlushService.performP\
  7089. reSyncedFlush(SyncedFlushService.java:414)
  7090.  
  7091. at org.elasticsearch.indices.flush.SyncedFlushService.access$1\
  7092. 000(SyncedFlushService.java:70)
  7093.  
  7094. at org.elasticsearch.indices.flush.SyncedFlushService$PreSynce\
  7095. dFlushTransportHandler.messageReceived(SyncedFlushService.java:696)
  7096.  
  7097. at org.elasticsearch.indices.flush.SyncedFlushService$PreSynce\
  7098. dFlushTransportHandler.messageReceived(SyncedFlushService.java:692)
  7099.  
  7100. at org.elasticsearch.transport.TransportRequestHandler.message\
  7101. Received(TransportRequestHandler.java:30)
  7102.  
  7103. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  7104. essageReceived(RequestHandlerRegistry.java:66)
  7105.  
  7106. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  7107. ortService.java:656)
  7108.  
  7109. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  7110. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  7111.  
  7112. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  7113. un(AbstractRunnable.java:37)
  7114.  
  7115. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7116. lExecutor.java:1149)
  7117.  
  7118. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7119. olExecutor.java:624)
  7120.  
  7121. at java.lang.Thread.run(Thread.java:748)
  7122.  
  7123.  
  7124.  
  7125. "elasticsearch[fmfe16][refresh][T#4]" "
  7126. #308 "
  7127. daemon "
  7128. prio=5 "
  7129. os_prio=15 "
  7130. tid=0x0000000b038cf800 "
  7131. nid=0x19a26 "
  7132. waiting on condition "
  7133. [0x00007fffcb6b5000]
  7134.  
  7135. java.lang.Thread.State: WAITING (parking)
  7136.  
  7137. at sun.misc.Unsafe.park(Native Method)
  7138.  
  7139. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  7140. rrent.locks.ReentrantLock$NonfairSync)
  7141.  
  7142. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7143. a:175)
  7144.  
  7145. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  7146. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  7147.  
  7148. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7149. reQueued(AbstractQueuedSynchronizer.java:870)
  7150.  
  7151. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7152. re(AbstractQueuedSynchronizer.java:1199)
  7153.  
  7154. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  7155. eentrantLock.java:209)
  7156.  
  7157. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  7158. .java:285)
  7159.  
  7160. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  7161. KeyedLock.java:76)
  7162.  
  7163. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  7164. iveVersionMap.java:431)
  7165.  
  7166. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  7167. ombstones(InternalEngine.java:1661)
  7168.  
  7169. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  7170. etedTombstones(InternalEngine.java:1348)
  7171.  
  7172. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  7173. nalEngine.java:1427)
  7174.  
  7175. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  7176. nalEngine.java:1383)
  7177.  
  7178. at org.elasticsearch.index.shard.IndexShard.refresh(IndexShard\
  7179. .java:861)
  7180.  
  7181. at org.elasticsearch.index.IndexService.maybeRefreshEngine(Ind\
  7182. exService.java:698)
  7183.  
  7184. at org.elasticsearch.index.IndexService.access$400(IndexServic\
  7185. e.java:97)
  7186.  
  7187. at org.elasticsearch.index.IndexService$AsyncRefreshTask.runIn\
  7188. ternal(IndexService.java:902)
  7189.  
  7190. at org.elasticsearch.index.IndexService$BaseAsyncTask.run(Inde\
  7191. xService.java:812)
  7192.  
  7193. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  7194. extPreservingRunnable.run(ThreadContext.java:566)
  7195.  
  7196. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7197. lExecutor.java:1149)
  7198.  
  7199. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7200. olExecutor.java:624)
  7201.  
  7202. at java.lang.Thread.run(Thread.java:748)
  7203.  
  7204.  
  7205.  
  7206. "elasticsearch[fmfe16][get][T#24]" "
  7207. #292 "
  7208. daemon "
  7209. prio=5 "
  7210. os_prio=15 "
  7211. tid=0x0000000afe053800 "
  7212. nid=0x199ca "
  7213. waiting on condition "
  7214. [0x00007fffcc6c5000]
  7215.  
  7216. java.lang.Thread.State: WAITING (parking)
  7217.  
  7218. at sun.misc.Unsafe.park(Native Method)
  7219.  
  7220. - parking to wait for <0x0000000873181870> (a java.util.concu\
  7221. rrent.LinkedTransferQueue)
  7222.  
  7223. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7224. a:175)
  7225.  
  7226. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7227. ransferQueue.java:737)
  7228.  
  7229. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7230. rQueue.java:647)
  7231.  
  7232. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7233. rQueue.java:1269)
  7234.  
  7235. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  7236. take(SizeBlockingQueue.java:161)
  7237.  
  7238. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7239. xecutor.java:1074)
  7240.  
  7241. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7242. lExecutor.java:1134)
  7243.  
  7244. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7245. olExecutor.java:624)
  7246.  
  7247. at java.lang.Thread.run(Thread.java:748)
  7248.  
  7249.  
  7250.  
  7251. "elasticsearch[fmfe16][get][T#23]" "
  7252. #291 "
  7253. daemon "
  7254. prio=5 "
  7255. os_prio=15 "
  7256. tid=0x0000000aff676000 "
  7257. nid=0x199c8 "
  7258. waiting on condition "
  7259. [0x00007fffcc7c6000]
  7260.  
  7261. java.lang.Thread.State: WAITING (parking)
  7262.  
  7263. at sun.misc.Unsafe.park(Native Method)
  7264.  
  7265. - parking to wait for <0x0000000873181870> (a java.util.concu\
  7266. rrent.LinkedTransferQueue)
  7267.  
  7268. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7269. a:175)
  7270.  
  7271. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7272. ransferQueue.java:737)
  7273.  
  7274. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7275. rQueue.java:647)
  7276.  
  7277. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7278. rQueue.java:1269)
  7279.  
  7280. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  7281. take(SizeBlockingQueue.java:161)
  7282.  
  7283. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7284. xecutor.java:1074)
  7285.  
  7286. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7287. lExecutor.java:1134)
  7288.  
  7289. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7290. olExecutor.java:624)
  7291.  
  7292. at java.lang.Thread.run(Thread.java:748)
  7293.  
  7294.  
  7295.  
  7296. "elasticsearch[fmfe16][get][T#22]" "
  7297. #290 "
  7298. daemon "
  7299. prio=5 "
  7300. os_prio=15 "
  7301. tid=0x0000000afc264800 "
  7302. nid=0x199c7 "
  7303. waiting on condition "
  7304. [0x00007fffcc8c7000]
  7305.  
  7306. java.lang.Thread.State: WAITING (parking)
  7307.  
  7308. at sun.misc.Unsafe.park(Native Method)
  7309.  
  7310. - parking to wait for <0x0000000873181870> (a java.util.concu\
  7311. rrent.LinkedTransferQueue)
  7312.  
  7313. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7314. a:175)
  7315.  
  7316. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7317. ransferQueue.java:737)
  7318.  
  7319. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7320. rQueue.java:647)
  7321.  
  7322. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7323. rQueue.java:1269)
  7324.  
  7325. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  7326. take(SizeBlockingQueue.java:161)
  7327.  
  7328. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7329. xecutor.java:1074)
  7330.  
  7331. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7332. lExecutor.java:1134)
  7333.  
  7334. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7335. olExecutor.java:624)
  7336.  
  7337. at java.lang.Thread.run(Thread.java:748)
  7338.  
  7339.  
  7340.  
  7341. "elasticsearch[fmfe16][bulk][T#1]" "
  7342. #289 "
  7343. daemon "
  7344. prio=5 "
  7345. os_prio=15 "
  7346. tid=0x0000000b04ce8000 "
  7347. nid=0x199c6 "
  7348. waiting on condition "
  7349. [0x00007fffcc9c8000]
  7350.  
  7351. java.lang.Thread.State: WAITING (parking)
  7352.  
  7353. at sun.misc.Unsafe.park(Native Method)
  7354.  
  7355. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  7356. rrent.locks.ReentrantLock$NonfairSync)
  7357.  
  7358. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7359. a:175)
  7360.  
  7361. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  7362. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  7363.  
  7364. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7365. reQueued(AbstractQueuedSynchronizer.java:870)
  7366.  
  7367. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7368. re(AbstractQueuedSynchronizer.java:1199)
  7369.  
  7370. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  7371. eentrantLock.java:209)
  7372.  
  7373. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  7374. .java:285)
  7375.  
  7376. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  7377. KeyedLock.java:76)
  7378.  
  7379. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  7380. iveVersionMap.java:431)
  7381.  
  7382. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  7383. ombstones(InternalEngine.java:1661)
  7384.  
  7385. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  7386. etedTombstones(InternalEngine.java:1348)
  7387.  
  7388. at org.elasticsearch.index.engine.InternalEngine.delete(Intern\
  7389. alEngine.java:1211)
  7390.  
  7391. at org.elasticsearch.index.shard.IndexShard.delete(IndexShard.\
  7392. java:839)
  7393.  
  7394. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  7395. on(IndexShard.java:806)
  7396.  
  7397. at org.elasticsearch.index.shard.IndexShard.applyDeleteOperati\
  7398. onOnPrimary(IndexShard.java:770)
  7399.  
  7400. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  7401. uteDeleteRequestOnPrimary(TransportShardBulkAction.java:562)
  7402.  
  7403. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  7404. uteDeleteRequest(TransportShardBulkAction.java:154)
  7405.  
  7406. at org.elasticsearch.action.bulk.TransportShardBulkAction.exec\
  7407. uteBulkItemRequest(TransportShardBulkAction.java:244)
  7408.  
  7409. at org.elasticsearch.action.bulk.TransportShardBulkAction.perf\
  7410. ormOnPrimary(TransportShardBulkAction.java:123)
  7411.  
  7412. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  7413. dOperationOnPrimary(TransportShardBulkAction.java:110)
  7414.  
  7415. at org.elasticsearch.action.bulk.TransportShardBulkAction.shar\
  7416. dOperationOnPrimary(TransportShardBulkAction.java:72)
  7417.  
  7418. at org.elasticsearch.action.support.replication.TransportRepli\
  7419. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  7420. java:1034)
  7421.  
  7422. at org.elasticsearch.action.support.replication.TransportRepli\
  7423. cationAction$PrimaryShardReference.perform(TransportReplicationAction.\
  7424. java:1012)
  7425.  
  7426. at org.elasticsearch.action.support.replication.ReplicationOpe\
  7427. ration.execute(ReplicationOperation.java:103)
  7428.  
  7429. at org.elasticsearch.action.support.replication.TransportRepli\
  7430. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  7431. java:359)
  7432.  
  7433. at org.elasticsearch.action.support.replication.TransportRepli\
  7434. cationAction$AsyncPrimaryAction.onResponse(TransportReplicationAction.\
  7435. java:299)
  7436.  
  7437. at org.elasticsearch.action.support.replication.TransportRepli\
  7438. cationAction$1.onResponse(TransportReplicationAction.java:975)
  7439.  
  7440. at org.elasticsearch.action.support.replication.TransportRepli\
  7441. cationAction$1.onResponse(TransportReplicationAction.java:972)
  7442.  
  7443. at org.elasticsearch.index.shard.IndexShardOperationPermits.ac\
  7444. quire(IndexShardOperationPermits.java:238)
  7445.  
  7446. at org.elasticsearch.index.shard.IndexShard.acquirePrimaryOper\
  7447. ationPermit(IndexShard.java:2220)
  7448.  
  7449. at org.elasticsearch.action.support.replication.TransportRepli\
  7450. cationAction.acquirePrimaryShardReference(TransportReplicationAction.j\
  7451. ava:984)
  7452.  
  7453. at org.elasticsearch.action.support.replication.TransportRepli\
  7454. cationAction.access$500(TransportReplicationAction.java:98)
  7455.  
  7456. at org.elasticsearch.action.support.replication.TransportRepli\
  7457. cationAction$AsyncPrimaryAction.doRun(TransportReplicationAction.java:\
  7458. 320)
  7459.  
  7460. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  7461. un(AbstractRunnable.java:37)
  7462.  
  7463. at org.elasticsearch.action.support.replication.TransportRepli\
  7464. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  7465. tReplicationAction.java:295)
  7466.  
  7467. at org.elasticsearch.action.support.replication.TransportRepli\
  7468. cationAction$PrimaryOperationTransportHandler.messageReceived(Transpor\
  7469. tReplicationAction.java:282)
  7470.  
  7471. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  7472. essageReceived(RequestHandlerRegistry.java:66)
  7473.  
  7474. at org.elasticsearch.transport.TransportService$7.doRun(Transp\
  7475. ortService.java:656)
  7476.  
  7477. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  7478. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  7479.  
  7480. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  7481. un(AbstractRunnable.java:37)
  7482.  
  7483. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7484. lExecutor.java:1149)
  7485.  
  7486. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7487. olExecutor.java:624)
  7488.  
  7489. at java.lang.Thread.run(Thread.java:748)
  7490.  
  7491.  
  7492.  
  7493. "elasticsearch[fmfe16][get][T#21]" "
  7494. #288 "
  7495. daemon "
  7496. prio=5 "
  7497. os_prio=15 "
  7498. tid=0x0000000afe4a7800 "
  7499. nid=0x199ad "
  7500. waiting on condition "
  7501. [0x00007fffccac9000]
  7502.  
  7503. java.lang.Thread.State: WAITING (parking)
  7504.  
  7505. at sun.misc.Unsafe.park(Native Method)
  7506.  
  7507. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  7508. rrent.locks.ReentrantLock$NonfairSync)
  7509.  
  7510. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7511. a:175)
  7512.  
  7513. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  7514. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  7515.  
  7516. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7517. reQueued(AbstractQueuedSynchronizer.java:870)
  7518.  
  7519. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7520. re(AbstractQueuedSynchronizer.java:1199)
  7521.  
  7522. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  7523. eentrantLock.java:209)
  7524.  
  7525. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  7526. .java:285)
  7527.  
  7528. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  7529. KeyedLock.java:76)
  7530.  
  7531. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  7532. iveVersionMap.java:431)
  7533.  
  7534. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  7535. ombstones(InternalEngine.java:1661)
  7536.  
  7537. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  7538. etedTombstones(InternalEngine.java:1348)
  7539.  
  7540. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  7541. nalEngine.java:1427)
  7542.  
  7543. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  7544. ngine.java:637)
  7545.  
  7546. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  7547. a:850)
  7548.  
  7549. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  7550. etService.java:153)
  7551.  
  7552. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  7553. vice.java:81)
  7554.  
  7555. at org.elasticsearch.action.get.TransportGetAction.shardOperat\
  7556. ion(TransportGetAction.java:88)
  7557.  
  7558. at org.elasticsearch.action.get.TransportGetAction.shardOperat\
  7559. ion(TransportGetAction.java:44)
  7560.  
  7561. at org.elasticsearch.action.support.single.shard.TransportSing\
  7562. leShardAction$ShardTransportHandler.messageReceived(TransportSingleSha\
  7563. rdAction.java:293)
  7564.  
  7565. at org.elasticsearch.action.support.single.shard.TransportSing\
  7566. leShardAction$ShardTransportHandler.messageReceived(TransportSingleSha\
  7567. rdAction.java:286)
  7568.  
  7569. at org.elasticsearch.transport.TransportRequestHandler.message\
  7570. Received(TransportRequestHandler.java:30)
  7571.  
  7572. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  7573. essageReceived(RequestHandlerRegistry.java:66)
  7574.  
  7575. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  7576. un(TcpTransport.java:1555)
  7577.  
  7578. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  7579. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  7580.  
  7581. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  7582. un(AbstractRunnable.java:37)
  7583.  
  7584. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7585. lExecutor.java:1149)
  7586.  
  7587. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7588. olExecutor.java:624)
  7589.  
  7590. at java.lang.Thread.run(Thread.java:748)
  7591.  
  7592.  
  7593.  
  7594. "elasticsearch[fmfe16][get][T#19]" "
  7595. #287 "
  7596. daemon "
  7597. prio=5 "
  7598. os_prio=15 "
  7599. tid=0x0000000aff675800 "
  7600. nid=0x199ac "
  7601. waiting on condition "
  7602. [0x00007fffccbca000]
  7603.  
  7604. java.lang.Thread.State: WAITING (parking)
  7605.  
  7606. at sun.misc.Unsafe.park(Native Method)
  7607.  
  7608. - parking to wait for <0x0000000873181870> (a java.util.concu\
  7609. rrent.LinkedTransferQueue)
  7610.  
  7611. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7612. a:175)
  7613.  
  7614. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7615. ransferQueue.java:737)
  7616.  
  7617. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7618. rQueue.java:647)
  7619.  
  7620. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7621. rQueue.java:1269)
  7622.  
  7623. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  7624. take(SizeBlockingQueue.java:161)
  7625.  
  7626. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7627. xecutor.java:1074)
  7628.  
  7629. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7630. lExecutor.java:1134)
  7631.  
  7632. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7633. olExecutor.java:624)
  7634.  
  7635. at java.lang.Thread.run(Thread.java:748)
  7636.  
  7637.  
  7638.  
  7639. "elasticsearch[fmfe16][get][T#20]" "
  7640. #286 "
  7641. daemon "
  7642. prio=5 "
  7643. os_prio=15 "
  7644. tid=0x0000000afc264000 "
  7645. nid=0x199a4 "
  7646. waiting on condition "
  7647. [0x00007fffccccb000]
  7648.  
  7649. java.lang.Thread.State: WAITING (parking)
  7650.  
  7651. at sun.misc.Unsafe.park(Native Method)
  7652.  
  7653. - parking to wait for <0x0000000873181870> (a java.util.concu\
  7654. rrent.LinkedTransferQueue)
  7655.  
  7656. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7657. a:175)
  7658.  
  7659. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7660. ransferQueue.java:737)
  7661.  
  7662. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7663. rQueue.java:647)
  7664.  
  7665. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7666. rQueue.java:1269)
  7667.  
  7668. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  7669. take(SizeBlockingQueue.java:161)
  7670.  
  7671. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7672. xecutor.java:1074)
  7673.  
  7674. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7675. lExecutor.java:1134)
  7676.  
  7677. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7678. olExecutor.java:624)
  7679.  
  7680. at java.lang.Thread.run(Thread.java:748)
  7681.  
  7682.  
  7683.  
  7684. "elasticsearch[fmfe16][get][T#18]" "
  7685. #285 "
  7686. daemon "
  7687. prio=5 "
  7688. os_prio=15 "
  7689. tid=0x0000000afc263000 "
  7690. nid=0x1999e "
  7691. waiting on condition "
  7692. [0x00007fffccdcc000]
  7693.  
  7694. java.lang.Thread.State: WAITING (parking)
  7695.  
  7696. at sun.misc.Unsafe.park(Native Method)
  7697.  
  7698. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  7699. rrent.locks.ReentrantLock$NonfairSync)
  7700.  
  7701. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7702. a:175)
  7703.  
  7704. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  7705. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  7706.  
  7707. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7708. reQueued(AbstractQueuedSynchronizer.java:870)
  7709.  
  7710. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7711. re(AbstractQueuedSynchronizer.java:1199)
  7712.  
  7713. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  7714. eentrantLock.java:209)
  7715.  
  7716. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  7717. .java:285)
  7718.  
  7719. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  7720. KeyedLock.java:76)
  7721.  
  7722. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  7723. iveVersionMap.java:431)
  7724.  
  7725. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  7726. ombstones(InternalEngine.java:1661)
  7727.  
  7728. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  7729. etedTombstones(InternalEngine.java:1348)
  7730.  
  7731. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  7732. nalEngine.java:1427)
  7733.  
  7734. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  7735. ngine.java:637)
  7736.  
  7737. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  7738. a:850)
  7739.  
  7740. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  7741. etService.java:153)
  7742.  
  7743. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  7744. vice.java:81)
  7745.  
  7746. at org.elasticsearch.action.get.TransportGetAction.shardOperat\
  7747. ion(TransportGetAction.java:88)
  7748.  
  7749. at org.elasticsearch.action.get.TransportGetAction.shardOperat\
  7750. ion(TransportGetAction.java:44)
  7751.  
  7752. at org.elasticsearch.action.support.single.shard.TransportSing\
  7753. leShardAction$ShardTransportHandler.messageReceived(TransportSingleSha\
  7754. rdAction.java:293)
  7755.  
  7756. at org.elasticsearch.action.support.single.shard.TransportSing\
  7757. leShardAction$ShardTransportHandler.messageReceived(TransportSingleSha\
  7758. rdAction.java:286)
  7759.  
  7760. at org.elasticsearch.transport.TransportRequestHandler.message\
  7761. Received(TransportRequestHandler.java:30)
  7762.  
  7763. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  7764. essageReceived(RequestHandlerRegistry.java:66)
  7765.  
  7766. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  7767. un(TcpTransport.java:1555)
  7768.  
  7769. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  7770. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  7771.  
  7772. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  7773. un(AbstractRunnable.java:37)
  7774.  
  7775. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7776. lExecutor.java:1149)
  7777.  
  7778. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7779. olExecutor.java:624)
  7780.  
  7781. at java.lang.Thread.run(Thread.java:748)
  7782.  
  7783.  
  7784.  
  7785. "elasticsearch[fmfe16][get][T#17]" "
  7786. #284 "
  7787. daemon "
  7788. prio=5 "
  7789. os_prio=15 "
  7790. tid=0x0000000afe4a7000 "
  7791. nid=0x1999c "
  7792. waiting on condition "
  7793. [0x00007fffccecd000]
  7794.  
  7795. java.lang.Thread.State: WAITING (parking)
  7796.  
  7797. at sun.misc.Unsafe.park(Native Method)
  7798.  
  7799. - parking to wait for <0x0000000873181870> (a java.util.concu\
  7800. rrent.LinkedTransferQueue)
  7801.  
  7802. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7803. a:175)
  7804.  
  7805. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7806. ransferQueue.java:737)
  7807.  
  7808. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7809. rQueue.java:647)
  7810.  
  7811. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7812. rQueue.java:1269)
  7813.  
  7814. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  7815. take(SizeBlockingQueue.java:161)
  7816.  
  7817. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7818. xecutor.java:1074)
  7819.  
  7820. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7821. lExecutor.java:1134)
  7822.  
  7823. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7824. olExecutor.java:624)
  7825.  
  7826. at java.lang.Thread.run(Thread.java:748)
  7827.  
  7828.  
  7829.  
  7830. "elasticsearch[fmfe16][get][T#16]" "
  7831. #283 "
  7832. daemon "
  7833. prio=5 "
  7834. os_prio=15 "
  7835. tid=0x0000000afe4a6000 "
  7836. nid=0x19995 "
  7837. waiting on condition "
  7838. [0x00007fffccfce000]
  7839.  
  7840. java.lang.Thread.State: WAITING (parking)
  7841.  
  7842. at sun.misc.Unsafe.park(Native Method)
  7843.  
  7844. - parking to wait for <0x0000000873181870> (a java.util.concu\
  7845. rrent.LinkedTransferQueue)
  7846.  
  7847. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7848. a:175)
  7849.  
  7850. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7851. ransferQueue.java:737)
  7852.  
  7853. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7854. rQueue.java:647)
  7855.  
  7856. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7857. rQueue.java:1269)
  7858.  
  7859. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  7860. take(SizeBlockingQueue.java:161)
  7861.  
  7862. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7863. xecutor.java:1074)
  7864.  
  7865. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7866. lExecutor.java:1134)
  7867.  
  7868. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7869. olExecutor.java:624)
  7870.  
  7871. at java.lang.Thread.run(Thread.java:748)
  7872.  
  7873.  
  7874.  
  7875. "elasticsearch[fmfe16][get][T#15]" "
  7876. #282 "
  7877. daemon "
  7878. prio=5 "
  7879. os_prio=15 "
  7880. tid=0x0000000aff674800 "
  7881. nid=0x1998d "
  7882. waiting on condition "
  7883. [0x00007fffcd0cf000]
  7884.  
  7885. java.lang.Thread.State: WAITING (parking)
  7886.  
  7887. at sun.misc.Unsafe.park(Native Method)
  7888.  
  7889. - parking to wait for <0x0000000873181870> (a java.util.concu\
  7890. rrent.LinkedTransferQueue)
  7891.  
  7892. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7893. a:175)
  7894.  
  7895. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  7896. ransferQueue.java:737)
  7897.  
  7898. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  7899. rQueue.java:647)
  7900.  
  7901. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  7902. rQueue.java:1269)
  7903.  
  7904. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  7905. take(SizeBlockingQueue.java:161)
  7906.  
  7907. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  7908. xecutor.java:1074)
  7909.  
  7910. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  7911. lExecutor.java:1134)
  7912.  
  7913. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  7914. olExecutor.java:624)
  7915.  
  7916. at java.lang.Thread.run(Thread.java:748)
  7917.  
  7918.  
  7919.  
  7920. "elasticsearch[fmfe16][get][T#14]" "
  7921. #281 "
  7922. daemon "
  7923. prio=5 "
  7924. os_prio=15 "
  7925. tid=0x0000000afc253800 "
  7926. nid=0x19985 "
  7927. waiting on condition "
  7928. [0x00007fffcd1d0000]
  7929.  
  7930. java.lang.Thread.State: WAITING (parking)
  7931.  
  7932. at sun.misc.Unsafe.park(Native Method)
  7933.  
  7934. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  7935. rrent.locks.ReentrantLock$NonfairSync)
  7936.  
  7937. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  7938. a:175)
  7939.  
  7940. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  7941. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  7942.  
  7943. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7944. reQueued(AbstractQueuedSynchronizer.java:870)
  7945.  
  7946. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  7947. re(AbstractQueuedSynchronizer.java:1199)
  7948.  
  7949. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  7950. eentrantLock.java:209)
  7951.  
  7952. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  7953. .java:285)
  7954.  
  7955. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  7956. KeyedLock.java:76)
  7957.  
  7958. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  7959. iveVersionMap.java:431)
  7960.  
  7961. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  7962. ombstones(InternalEngine.java:1661)
  7963.  
  7964. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  7965. etedTombstones(InternalEngine.java:1348)
  7966.  
  7967. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  7968. nalEngine.java:1427)
  7969.  
  7970. at org.elasticsearch.index.engine.InternalEngine.getVersionFro\
  7971. mMap(InternalEngine.java:737)
  7972.  
  7973. - locked <0x00000008b40309c8> "
  7974. (a org.elasticsearch.index.engine.LiveVersionMap)
  7975.  
  7976. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  7977. ngine.java:627)
  7978.  
  7979. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  7980. a:850)
  7981.  
  7982. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  7983. etService.java:153)
  7984.  
  7985. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  7986. vice.java:81)
  7987.  
  7988. at org.elasticsearch.action.get.TransportGetAction.shardOperat\
  7989. ion(TransportGetAction.java:88)
  7990.  
  7991. at org.elasticsearch.action.get.TransportGetAction.shardOperat\
  7992. ion(TransportGetAction.java:44)
  7993.  
  7994. at org.elasticsearch.action.support.single.shard.TransportSing\
  7995. leShardAction$ShardTransportHandler.messageReceived(TransportSingleSha\
  7996. rdAction.java:293)
  7997.  
  7998. at org.elasticsearch.action.support.single.shard.TransportSing\
  7999. leShardAction$ShardTransportHandler.messageReceived(TransportSingleSha\
  8000. rdAction.java:286)
  8001.  
  8002. at org.elasticsearch.transport.TransportRequestHandler.message\
  8003. Received(TransportRequestHandler.java:30)
  8004.  
  8005. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  8006. essageReceived(RequestHandlerRegistry.java:66)
  8007.  
  8008. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  8009. un(TcpTransport.java:1555)
  8010.  
  8011. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  8012. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  8013.  
  8014. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  8015. un(AbstractRunnable.java:37)
  8016.  
  8017. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8018. lExecutor.java:1149)
  8019.  
  8020. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8021. olExecutor.java:624)
  8022.  
  8023. at java.lang.Thread.run(Thread.java:748)
  8024.  
  8025.  
  8026.  
  8027. "elasticsearch[fmfe16][get][T#13]" "
  8028. #280 "
  8029. daemon "
  8030. prio=5 "
  8031. os_prio=15 "
  8032. tid=0x0000000afc252800 "
  8033. nid=0x1997d "
  8034. waiting on condition "
  8035. [0x00007fffcd2d1000]
  8036.  
  8037. java.lang.Thread.State: WAITING (parking)
  8038.  
  8039. at sun.misc.Unsafe.park(Native Method)
  8040.  
  8041. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8042. rrent.LinkedTransferQueue)
  8043.  
  8044. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8045. a:175)
  8046.  
  8047. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8048. ransferQueue.java:737)
  8049.  
  8050. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8051. rQueue.java:647)
  8052.  
  8053. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8054. rQueue.java:1269)
  8055.  
  8056. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8057. take(SizeBlockingQueue.java:161)
  8058.  
  8059. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8060. xecutor.java:1074)
  8061.  
  8062. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8063. lExecutor.java:1134)
  8064.  
  8065. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8066. olExecutor.java:624)
  8067.  
  8068. at java.lang.Thread.run(Thread.java:748)
  8069.  
  8070.  
  8071.  
  8072. "elasticsearch[fmfe16][get][T#12]" "
  8073. #279 "
  8074. daemon "
  8075. prio=5 "
  8076. os_prio=15 "
  8077. tid=0x0000000b00aa3800 "
  8078. nid=0x1997c "
  8079. waiting on condition "
  8080. [0x00007fffcd3d2000]
  8081.  
  8082. java.lang.Thread.State: WAITING (parking)
  8083.  
  8084. at sun.misc.Unsafe.park(Native Method)
  8085.  
  8086. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8087. rrent.LinkedTransferQueue)
  8088.  
  8089. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8090. a:175)
  8091.  
  8092. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8093. ransferQueue.java:737)
  8094.  
  8095. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8096. rQueue.java:647)
  8097.  
  8098. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8099. rQueue.java:1269)
  8100.  
  8101. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8102. take(SizeBlockingQueue.java:161)
  8103.  
  8104. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8105. xecutor.java:1074)
  8106.  
  8107. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8108. lExecutor.java:1134)
  8109.  
  8110. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8111. olExecutor.java:624)
  8112.  
  8113. at java.lang.Thread.run(Thread.java:748)
  8114.  
  8115.  
  8116.  
  8117. "elasticsearch[fmfe16][get][T#11]" "
  8118. #278 "
  8119. daemon "
  8120. prio=5 "
  8121. os_prio=15 "
  8122. tid=0x0000000b00aa3000 "
  8123. nid=0x19975 "
  8124. waiting on condition "
  8125. [0x00007fffcd4d3000]
  8126.  
  8127. java.lang.Thread.State: WAITING (parking)
  8128.  
  8129. at sun.misc.Unsafe.park(Native Method)
  8130.  
  8131. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8132. rrent.LinkedTransferQueue)
  8133.  
  8134. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8135. a:175)
  8136.  
  8137. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8138. ransferQueue.java:737)
  8139.  
  8140. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8141. rQueue.java:647)
  8142.  
  8143. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8144. rQueue.java:1269)
  8145.  
  8146. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8147. take(SizeBlockingQueue.java:161)
  8148.  
  8149. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8150. xecutor.java:1074)
  8151.  
  8152. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8153. lExecutor.java:1134)
  8154.  
  8155. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8156. olExecutor.java:624)
  8157.  
  8158. at java.lang.Thread.run(Thread.java:748)
  8159.  
  8160.  
  8161.  
  8162. "elasticsearch[fmfe16][get][T#10]" "
  8163. #277 "
  8164. daemon "
  8165. prio=5 "
  8166. os_prio=15 "
  8167. tid=0x0000000afe4a5800 "
  8168. nid=0x19970 "
  8169. waiting on condition "
  8170. [0x00007fffcd5d4000]
  8171.  
  8172. java.lang.Thread.State: WAITING (parking)
  8173.  
  8174. at sun.misc.Unsafe.park(Native Method)
  8175.  
  8176. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8177. rrent.LinkedTransferQueue)
  8178.  
  8179. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8180. a:175)
  8181.  
  8182. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8183. ransferQueue.java:737)
  8184.  
  8185. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8186. rQueue.java:647)
  8187.  
  8188. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8189. rQueue.java:1269)
  8190.  
  8191. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8192. take(SizeBlockingQueue.java:161)
  8193.  
  8194. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8195. xecutor.java:1074)
  8196.  
  8197. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8198. lExecutor.java:1134)
  8199.  
  8200. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8201. olExecutor.java:624)
  8202.  
  8203. at java.lang.Thread.run(Thread.java:748)
  8204.  
  8205.  
  8206.  
  8207. "elasticsearch[fmfe16][get][T#9]" "
  8208. #276 "
  8209. daemon "
  8210. prio=5 "
  8211. os_prio=15 "
  8212. tid=0x0000000afe052800 "
  8213. nid=0x18810 "
  8214. waiting on condition "
  8215. [0x00007fffcd6d5000]
  8216.  
  8217. java.lang.Thread.State: WAITING (parking)
  8218.  
  8219. at sun.misc.Unsafe.park(Native Method)
  8220.  
  8221. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8222. rrent.LinkedTransferQueue)
  8223.  
  8224. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8225. a:175)
  8226.  
  8227. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8228. ransferQueue.java:737)
  8229.  
  8230. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8231. rQueue.java:647)
  8232.  
  8233. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8234. rQueue.java:1269)
  8235.  
  8236. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8237. take(SizeBlockingQueue.java:161)
  8238.  
  8239. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8240. xecutor.java:1074)
  8241.  
  8242. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8243. lExecutor.java:1134)
  8244.  
  8245. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8246. olExecutor.java:624)
  8247.  
  8248. at java.lang.Thread.run(Thread.java:748)
  8249.  
  8250.  
  8251.  
  8252. "elasticsearch[fmfe16][get][T#8]" "
  8253. #274 "
  8254. daemon "
  8255. prio=5 "
  8256. os_prio=15 "
  8257. tid=0x0000000afe4a4800 "
  8258. nid=0x19969 "
  8259. waiting on condition "
  8260. [0x00007fffcd7d6000]
  8261.  
  8262. java.lang.Thread.State: WAITING (parking)
  8263.  
  8264. at sun.misc.Unsafe.park(Native Method)
  8265.  
  8266. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8267. rrent.LinkedTransferQueue)
  8268.  
  8269. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8270. a:175)
  8271.  
  8272. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8273. ransferQueue.java:737)
  8274.  
  8275. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8276. rQueue.java:647)
  8277.  
  8278. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8279. rQueue.java:1269)
  8280.  
  8281. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8282. take(SizeBlockingQueue.java:161)
  8283.  
  8284. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8285. xecutor.java:1074)
  8286.  
  8287. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8288. lExecutor.java:1134)
  8289.  
  8290. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8291. olExecutor.java:624)
  8292.  
  8293. at java.lang.Thread.run(Thread.java:748)
  8294.  
  8295.  
  8296.  
  8297. "elasticsearch[fmfe16][get][T#7]" "
  8298. #272 "
  8299. daemon "
  8300. prio=5 "
  8301. os_prio=15 "
  8302. tid=0x0000000b0004b000 "
  8303. nid=0x19962 "
  8304. waiting on condition "
  8305. [0x00007fffcdad9000]
  8306.  
  8307. java.lang.Thread.State: WAITING (parking)
  8308.  
  8309. at sun.misc.Unsafe.park(Native Method)
  8310.  
  8311. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8312. rrent.LinkedTransferQueue)
  8313.  
  8314. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8315. a:175)
  8316.  
  8317. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8318. ransferQueue.java:737)
  8319.  
  8320. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8321. rQueue.java:647)
  8322.  
  8323. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8324. rQueue.java:1269)
  8325.  
  8326. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8327. take(SizeBlockingQueue.java:161)
  8328.  
  8329. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8330. xecutor.java:1074)
  8331.  
  8332. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8333. lExecutor.java:1134)
  8334.  
  8335. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8336. olExecutor.java:624)
  8337.  
  8338. at java.lang.Thread.run(Thread.java:748)
  8339.  
  8340.  
  8341.  
  8342. "elasticsearch[fmfe16][get][T#6]" "
  8343. #271 "
  8344. daemon "
  8345. prio=5 "
  8346. os_prio=15 "
  8347. tid=0x0000000aff674000 "
  8348. nid=0x19959 "
  8349. waiting on condition "
  8350. [0x00007fffcdbda000]
  8351.  
  8352. java.lang.Thread.State: WAITING (parking)
  8353.  
  8354. at sun.misc.Unsafe.park(Native Method)
  8355.  
  8356. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8357. rrent.LinkedTransferQueue)
  8358.  
  8359. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8360. a:175)
  8361.  
  8362. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8363. ransferQueue.java:737)
  8364.  
  8365. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8366. rQueue.java:647)
  8367.  
  8368. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8369. rQueue.java:1269)
  8370.  
  8371. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8372. take(SizeBlockingQueue.java:161)
  8373.  
  8374. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8375. xecutor.java:1074)
  8376.  
  8377. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8378. lExecutor.java:1134)
  8379.  
  8380. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8381. olExecutor.java:624)
  8382.  
  8383. at java.lang.Thread.run(Thread.java:748)
  8384.  
  8385.  
  8386.  
  8387. "elasticsearch[fmfe16][get][T#5]" "
  8388. #269 "
  8389. daemon "
  8390. prio=5 "
  8391. os_prio=15 "
  8392. tid=0x0000000afc252000 "
  8393. nid=0x1994d "
  8394. waiting on condition "
  8395. [0x00007fffcdddc000]
  8396.  
  8397. java.lang.Thread.State: WAITING (parking)
  8398.  
  8399. at sun.misc.Unsafe.park(Native Method)
  8400.  
  8401. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8402. rrent.LinkedTransferQueue)
  8403.  
  8404. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8405. a:175)
  8406.  
  8407. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8408. ransferQueue.java:737)
  8409.  
  8410. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8411. rQueue.java:647)
  8412.  
  8413. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8414. rQueue.java:1269)
  8415.  
  8416. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8417. take(SizeBlockingQueue.java:161)
  8418.  
  8419. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8420. xecutor.java:1074)
  8421.  
  8422. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8423. lExecutor.java:1134)
  8424.  
  8425. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8426. olExecutor.java:624)
  8427.  
  8428. at java.lang.Thread.run(Thread.java:748)
  8429.  
  8430.  
  8431.  
  8432. "elasticsearch[fmfe16][get][T#4]" "
  8433. #268 "
  8434. daemon "
  8435. prio=5 "
  8436. os_prio=15 "
  8437. tid=0x0000000b0004a000 "
  8438. nid=0x19942 "
  8439. waiting on condition "
  8440. [0x00007fffcdedd000]
  8441.  
  8442. java.lang.Thread.State: WAITING (parking)
  8443.  
  8444. at sun.misc.Unsafe.park(Native Method)
  8445.  
  8446. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8447. rrent.LinkedTransferQueue)
  8448.  
  8449. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8450. a:175)
  8451.  
  8452. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8453. ransferQueue.java:737)
  8454.  
  8455. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8456. rQueue.java:647)
  8457.  
  8458. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8459. rQueue.java:1269)
  8460.  
  8461. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8462. take(SizeBlockingQueue.java:161)
  8463.  
  8464. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8465. xecutor.java:1074)
  8466.  
  8467. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8468. lExecutor.java:1134)
  8469.  
  8470. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8471. olExecutor.java:624)
  8472.  
  8473. at java.lang.Thread.run(Thread.java:748)
  8474.  
  8475.  
  8476.  
  8477. "elasticsearch[fmfe16][get][T#3]" "
  8478. #267 "
  8479. daemon "
  8480. prio=5 "
  8481. os_prio=15 "
  8482. tid=0x0000000aff673000 "
  8483. nid=0x19938 "
  8484. waiting on condition "
  8485. [0x00007fffcdfde000]
  8486.  
  8487. java.lang.Thread.State: WAITING (parking)
  8488.  
  8489. at sun.misc.Unsafe.park(Native Method)
  8490.  
  8491. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8492. rrent.LinkedTransferQueue)
  8493.  
  8494. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8495. a:175)
  8496.  
  8497. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8498. ransferQueue.java:737)
  8499.  
  8500. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8501. rQueue.java:647)
  8502.  
  8503. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8504. rQueue.java:1269)
  8505.  
  8506. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8507. take(SizeBlockingQueue.java:161)
  8508.  
  8509. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8510. xecutor.java:1074)
  8511.  
  8512. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8513. lExecutor.java:1134)
  8514.  
  8515. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8516. olExecutor.java:624)
  8517.  
  8518. at java.lang.Thread.run(Thread.java:748)
  8519.  
  8520.  
  8521.  
  8522. "elasticsearch[fmfe16][get][T#2]" "
  8523. #266 "
  8524. daemon "
  8525. prio=5 "
  8526. os_prio=15 "
  8527. tid=0x0000000afe052000 "
  8528. nid=0x19935 "
  8529. waiting on condition "
  8530. [0x00007fffce0df000]
  8531.  
  8532. java.lang.Thread.State: WAITING (parking)
  8533.  
  8534. at sun.misc.Unsafe.park(Native Method)
  8535.  
  8536. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8537. rrent.LinkedTransferQueue)
  8538.  
  8539. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8540. a:175)
  8541.  
  8542. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8543. ransferQueue.java:737)
  8544.  
  8545. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8546. rQueue.java:647)
  8547.  
  8548. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8549. rQueue.java:1269)
  8550.  
  8551. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8552. take(SizeBlockingQueue.java:161)
  8553.  
  8554. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8555. xecutor.java:1074)
  8556.  
  8557. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8558. lExecutor.java:1134)
  8559.  
  8560. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8561. olExecutor.java:624)
  8562.  
  8563. at java.lang.Thread.run(Thread.java:748)
  8564.  
  8565.  
  8566.  
  8567. "elasticsearch[fmfe16][get][T#1]" "
  8568. #264 "
  8569. daemon "
  8570. prio=5 "
  8571. os_prio=15 "
  8572. tid=0x0000000afc251000 "
  8573. nid=0x19930 "
  8574. waiting on condition "
  8575. [0x00007fffce2e1000]
  8576.  
  8577. java.lang.Thread.State: WAITING (parking)
  8578.  
  8579. at sun.misc.Unsafe.park(Native Method)
  8580.  
  8581. - parking to wait for <0x0000000873181870> (a java.util.concu\
  8582. rrent.LinkedTransferQueue)
  8583.  
  8584. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8585. a:175)
  8586.  
  8587. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8588. ransferQueue.java:737)
  8589.  
  8590. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8591. rQueue.java:647)
  8592.  
  8593. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8594. rQueue.java:1269)
  8595.  
  8596. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8597. take(SizeBlockingQueue.java:161)
  8598.  
  8599. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8600. xecutor.java:1074)
  8601.  
  8602. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8603. lExecutor.java:1134)
  8604.  
  8605. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8606. olExecutor.java:624)
  8607.  
  8608. at java.lang.Thread.run(Thread.java:748)
  8609.  
  8610.  
  8611.  
  8612. "elasticsearch[fmfe16][search][T#37]" "
  8613. #248 "
  8614. daemon "
  8615. prio=5 "
  8616. os_prio=15 "
  8617. tid=0x0000000b03596800 "
  8618. nid=0x198f3 "
  8619. waiting on condition "
  8620. [0x00007fffcf2f1000]
  8621.  
  8622. java.lang.Thread.State: WAITING (parking)
  8623.  
  8624. at sun.misc.Unsafe.park(Native Method)
  8625.  
  8626. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8627. rrent.LinkedTransferQueue)
  8628.  
  8629. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8630. a:175)
  8631.  
  8632. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8633. ransferQueue.java:737)
  8634.  
  8635. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8636. rQueue.java:647)
  8637.  
  8638. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8639. rQueue.java:1269)
  8640.  
  8641. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8642. take(SizeBlockingQueue.java:161)
  8643.  
  8644. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8645. xecutor.java:1074)
  8646.  
  8647. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8648. lExecutor.java:1134)
  8649.  
  8650. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8651. olExecutor.java:624)
  8652.  
  8653. at java.lang.Thread.run(Thread.java:748)
  8654.  
  8655.  
  8656.  
  8657. "elasticsearch[fmfe16][search][T#36]" "
  8658. #247 "
  8659. daemon "
  8660. prio=5 "
  8661. os_prio=15 "
  8662. tid=0x0000000b03595800 "
  8663. nid=0x198f2 "
  8664. waiting on condition "
  8665. [0x00007fffcf3f2000]
  8666.  
  8667. java.lang.Thread.State: WAITING (parking)
  8668.  
  8669. at sun.misc.Unsafe.park(Native Method)
  8670.  
  8671. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8672. rrent.LinkedTransferQueue)
  8673.  
  8674. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8675. a:175)
  8676.  
  8677. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8678. ransferQueue.java:737)
  8679.  
  8680. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8681. rQueue.java:647)
  8682.  
  8683. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8684. rQueue.java:1269)
  8685.  
  8686. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8687. take(SizeBlockingQueue.java:161)
  8688.  
  8689. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8690. xecutor.java:1074)
  8691.  
  8692. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8693. lExecutor.java:1134)
  8694.  
  8695. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8696. olExecutor.java:624)
  8697.  
  8698. at java.lang.Thread.run(Thread.java:748)
  8699.  
  8700.  
  8701.  
  8702. "elasticsearch[fmfe16][search][T#35]" "
  8703. #246 "
  8704. daemon "
  8705. prio=5 "
  8706. os_prio=15 "
  8707. tid=0x0000000b03595000 "
  8708. nid=0x198f0 "
  8709. waiting on condition "
  8710. [0x00007fffcf4f3000]
  8711.  
  8712. java.lang.Thread.State: WAITING (parking)
  8713.  
  8714. at sun.misc.Unsafe.park(Native Method)
  8715.  
  8716. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8717. rrent.LinkedTransferQueue)
  8718.  
  8719. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8720. a:175)
  8721.  
  8722. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8723. ransferQueue.java:737)
  8724.  
  8725. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8726. rQueue.java:647)
  8727.  
  8728. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8729. rQueue.java:1269)
  8730.  
  8731. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8732. take(SizeBlockingQueue.java:161)
  8733.  
  8734. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8735. xecutor.java:1074)
  8736.  
  8737. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8738. lExecutor.java:1134)
  8739.  
  8740. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8741. olExecutor.java:624)
  8742.  
  8743. at java.lang.Thread.run(Thread.java:748)
  8744.  
  8745.  
  8746.  
  8747. "elasticsearch[fmfe16][search][T#34]" "
  8748. #245 "
  8749. daemon "
  8750. prio=5 "
  8751. os_prio=15 "
  8752. tid=0x0000000b03594000 "
  8753. nid=0x198ea "
  8754. waiting on condition "
  8755. [0x00007fffcf5f4000]
  8756.  
  8757. java.lang.Thread.State: WAITING (parking)
  8758.  
  8759. at sun.misc.Unsafe.park(Native Method)
  8760.  
  8761. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8762. rrent.LinkedTransferQueue)
  8763.  
  8764. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8765. a:175)
  8766.  
  8767. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8768. ransferQueue.java:737)
  8769.  
  8770. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8771. rQueue.java:647)
  8772.  
  8773. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8774. rQueue.java:1269)
  8775.  
  8776. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8777. take(SizeBlockingQueue.java:161)
  8778.  
  8779. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8780. xecutor.java:1074)
  8781.  
  8782. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8783. lExecutor.java:1134)
  8784.  
  8785. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8786. olExecutor.java:624)
  8787.  
  8788. at java.lang.Thread.run(Thread.java:748)
  8789.  
  8790.  
  8791.  
  8792. "elasticsearch[fmfe16][search][T#33]" "
  8793. #244 "
  8794. daemon "
  8795. prio=5 "
  8796. os_prio=15 "
  8797. tid=0x0000000b03593800 "
  8798. nid=0x198df "
  8799. waiting on condition "
  8800. [0x00007fffcf6f5000]
  8801.  
  8802. java.lang.Thread.State: WAITING (parking)
  8803.  
  8804. at sun.misc.Unsafe.park(Native Method)
  8805.  
  8806. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8807. rrent.LinkedTransferQueue)
  8808.  
  8809. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8810. a:175)
  8811.  
  8812. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8813. ransferQueue.java:737)
  8814.  
  8815. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8816. rQueue.java:647)
  8817.  
  8818. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8819. rQueue.java:1269)
  8820.  
  8821. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8822. take(SizeBlockingQueue.java:161)
  8823.  
  8824. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8825. xecutor.java:1074)
  8826.  
  8827. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8828. lExecutor.java:1134)
  8829.  
  8830. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8831. olExecutor.java:624)
  8832.  
  8833. at java.lang.Thread.run(Thread.java:748)
  8834.  
  8835.  
  8836.  
  8837. "elasticsearch[fmfe16][search][T#32]" "
  8838. #243 "
  8839. daemon "
  8840. prio=5 "
  8841. os_prio=15 "
  8842. tid=0x0000000b03592800 "
  8843. nid=0x198de "
  8844. waiting on condition "
  8845. [0x00007fffcf7f6000]
  8846.  
  8847. java.lang.Thread.State: WAITING (parking)
  8848.  
  8849. at sun.misc.Unsafe.park(Native Method)
  8850.  
  8851. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8852. rrent.LinkedTransferQueue)
  8853.  
  8854. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8855. a:175)
  8856.  
  8857. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8858. ransferQueue.java:737)
  8859.  
  8860. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8861. rQueue.java:647)
  8862.  
  8863. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8864. rQueue.java:1269)
  8865.  
  8866. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8867. take(SizeBlockingQueue.java:161)
  8868.  
  8869. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8870. xecutor.java:1074)
  8871.  
  8872. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8873. lExecutor.java:1134)
  8874.  
  8875. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8876. olExecutor.java:624)
  8877.  
  8878. at java.lang.Thread.run(Thread.java:748)
  8879.  
  8880.  
  8881.  
  8882. "elasticsearch[fmfe16][search][T#31]" "
  8883. #242 "
  8884. daemon "
  8885. prio=5 "
  8886. os_prio=15 "
  8887. tid=0x0000000b03574800 "
  8888. nid=0x198d9 "
  8889. waiting on condition "
  8890. [0x00007fffcf8f7000]
  8891.  
  8892. java.lang.Thread.State: WAITING (parking)
  8893.  
  8894. at sun.misc.Unsafe.park(Native Method)
  8895.  
  8896. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8897. rrent.LinkedTransferQueue)
  8898.  
  8899. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8900. a:175)
  8901.  
  8902. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8903. ransferQueue.java:737)
  8904.  
  8905. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8906. rQueue.java:647)
  8907.  
  8908. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8909. rQueue.java:1269)
  8910.  
  8911. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8912. take(SizeBlockingQueue.java:161)
  8913.  
  8914. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8915. xecutor.java:1074)
  8916.  
  8917. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8918. lExecutor.java:1134)
  8919.  
  8920. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8921. olExecutor.java:624)
  8922.  
  8923. at java.lang.Thread.run(Thread.java:748)
  8924.  
  8925.  
  8926.  
  8927. "elasticsearch[fmfe16][search][T#30]" "
  8928. #241 "
  8929. daemon "
  8930. prio=5 "
  8931. os_prio=15 "
  8932. tid=0x0000000b03591000 "
  8933. nid=0x198d7 "
  8934. waiting on condition "
  8935. [0x00007fffcf9f8000]
  8936.  
  8937. java.lang.Thread.State: WAITING (parking)
  8938.  
  8939. at sun.misc.Unsafe.park(Native Method)
  8940.  
  8941. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8942. rrent.LinkedTransferQueue)
  8943.  
  8944. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8945. a:175)
  8946.  
  8947. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8948. ransferQueue.java:737)
  8949.  
  8950. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8951. rQueue.java:647)
  8952.  
  8953. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8954. rQueue.java:1269)
  8955.  
  8956. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  8957. take(SizeBlockingQueue.java:161)
  8958.  
  8959. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  8960. xecutor.java:1074)
  8961.  
  8962. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  8963. lExecutor.java:1134)
  8964.  
  8965. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  8966. olExecutor.java:624)
  8967.  
  8968. at java.lang.Thread.run(Thread.java:748)
  8969.  
  8970.  
  8971.  
  8972. "elasticsearch[fmfe16][search][T#29]" "
  8973. #240 "
  8974. daemon "
  8975. prio=5 "
  8976. os_prio=15 "
  8977. tid=0x0000000b0358a800 "
  8978. nid=0x198d2 "
  8979. waiting on condition "
  8980. [0x00007fffcfaf9000]
  8981.  
  8982. java.lang.Thread.State: WAITING (parking)
  8983.  
  8984. at sun.misc.Unsafe.park(Native Method)
  8985.  
  8986. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  8987. rrent.LinkedTransferQueue)
  8988.  
  8989. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  8990. a:175)
  8991.  
  8992. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  8993. ransferQueue.java:737)
  8994.  
  8995. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  8996. rQueue.java:647)
  8997.  
  8998. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  8999. rQueue.java:1269)
  9000.  
  9001. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9002. take(SizeBlockingQueue.java:161)
  9003.  
  9004. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9005. xecutor.java:1074)
  9006.  
  9007. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9008. lExecutor.java:1134)
  9009.  
  9010. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9011. olExecutor.java:624)
  9012.  
  9013. at java.lang.Thread.run(Thread.java:748)
  9014.  
  9015.  
  9016.  
  9017. "elasticsearch[fmfe16][search][T#28]" "
  9018. #239 "
  9019. daemon "
  9020. prio=5 "
  9021. os_prio=15 "
  9022. tid=0x0000000b03589800 "
  9023. nid=0x198cc "
  9024. waiting on condition "
  9025. [0x00007fffcfbfa000]
  9026.  
  9027. java.lang.Thread.State: WAITING (parking)
  9028.  
  9029. at sun.misc.Unsafe.park(Native Method)
  9030.  
  9031. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9032. rrent.LinkedTransferQueue)
  9033.  
  9034. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9035. a:175)
  9036.  
  9037. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9038. ransferQueue.java:737)
  9039.  
  9040. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9041. rQueue.java:647)
  9042.  
  9043. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9044. rQueue.java:1269)
  9045.  
  9046. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9047. take(SizeBlockingQueue.java:161)
  9048.  
  9049. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9050. xecutor.java:1074)
  9051.  
  9052. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9053. lExecutor.java:1134)
  9054.  
  9055. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9056. olExecutor.java:624)
  9057.  
  9058. at java.lang.Thread.run(Thread.java:748)
  9059.  
  9060.  
  9061.  
  9062. "elasticsearch[fmfe16][search][T#27]" "
  9063. #238 "
  9064. daemon "
  9065. prio=5 "
  9066. os_prio=15 "
  9067. tid=0x0000000b03589000 "
  9068. nid=0x198c3 "
  9069. waiting on condition "
  9070. [0x00007fffcfcfb000]
  9071.  
  9072. java.lang.Thread.State: WAITING (parking)
  9073.  
  9074. at sun.misc.Unsafe.park(Native Method)
  9075.  
  9076. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9077. rrent.LinkedTransferQueue)
  9078.  
  9079. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9080. a:175)
  9081.  
  9082. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9083. ransferQueue.java:737)
  9084.  
  9085. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9086. rQueue.java:647)
  9087.  
  9088. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9089. rQueue.java:1269)
  9090.  
  9091. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9092. take(SizeBlockingQueue.java:161)
  9093.  
  9094. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9095. xecutor.java:1074)
  9096.  
  9097. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9098. lExecutor.java:1134)
  9099.  
  9100. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9101. olExecutor.java:624)
  9102.  
  9103. at java.lang.Thread.run(Thread.java:748)
  9104.  
  9105.  
  9106.  
  9107. "elasticsearch[fmfe16][search][T#26]" "
  9108. #237 "
  9109. daemon "
  9110. prio=5 "
  9111. os_prio=15 "
  9112. tid=0x0000000b03588000 "
  9113. nid=0x198c2 "
  9114. waiting on condition "
  9115. [0x00007fffcfdfc000]
  9116.  
  9117. java.lang.Thread.State: WAITING (parking)
  9118.  
  9119. at sun.misc.Unsafe.park(Native Method)
  9120.  
  9121. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9122. rrent.LinkedTransferQueue)
  9123.  
  9124. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9125. a:175)
  9126.  
  9127. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9128. ransferQueue.java:737)
  9129.  
  9130. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9131. rQueue.java:647)
  9132.  
  9133. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9134. rQueue.java:1269)
  9135.  
  9136. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9137. take(SizeBlockingQueue.java:161)
  9138.  
  9139. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9140. xecutor.java:1074)
  9141.  
  9142. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9143. lExecutor.java:1134)
  9144.  
  9145. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9146. olExecutor.java:624)
  9147.  
  9148. at java.lang.Thread.run(Thread.java:748)
  9149.  
  9150.  
  9151.  
  9152. "elasticsearch[fmfe16][search][T#25]" "
  9153. #236 "
  9154. daemon "
  9155. prio=5 "
  9156. os_prio=15 "
  9157. tid=0x0000000b03587800 "
  9158. nid=0x198b6 "
  9159. waiting on condition "
  9160. [0x00007fffcfefd000]
  9161.  
  9162. java.lang.Thread.State: WAITING (parking)
  9163.  
  9164. at sun.misc.Unsafe.park(Native Method)
  9165.  
  9166. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9167. rrent.LinkedTransferQueue)
  9168.  
  9169. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9170. a:175)
  9171.  
  9172. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9173. ransferQueue.java:737)
  9174.  
  9175. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9176. rQueue.java:647)
  9177.  
  9178. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9179. rQueue.java:1269)
  9180.  
  9181. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9182. take(SizeBlockingQueue.java:161)
  9183.  
  9184. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9185. xecutor.java:1074)
  9186.  
  9187. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9188. lExecutor.java:1134)
  9189.  
  9190. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9191. olExecutor.java:624)
  9192.  
  9193. at java.lang.Thread.run(Thread.java:748)
  9194.  
  9195.  
  9196.  
  9197. "elasticsearch[fmfe16][search][T#24]" "
  9198. #235 "
  9199. daemon "
  9200. prio=5 "
  9201. os_prio=15 "
  9202. tid=0x0000000b03586800 "
  9203. nid=0x198b5 "
  9204. waiting on condition "
  9205. [0x00007fffcfffe000]
  9206.  
  9207. java.lang.Thread.State: WAITING (parking)
  9208.  
  9209. at sun.misc.Unsafe.park(Native Method)
  9210.  
  9211. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9212. rrent.LinkedTransferQueue)
  9213.  
  9214. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9215. a:175)
  9216.  
  9217. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9218. ransferQueue.java:737)
  9219.  
  9220. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9221. rQueue.java:647)
  9222.  
  9223. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9224. rQueue.java:1269)
  9225.  
  9226. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9227. take(SizeBlockingQueue.java:161)
  9228.  
  9229. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9230. xecutor.java:1074)
  9231.  
  9232. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9233. lExecutor.java:1134)
  9234.  
  9235. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9236. olExecutor.java:624)
  9237.  
  9238. at java.lang.Thread.run(Thread.java:748)
  9239.  
  9240.  
  9241.  
  9242. "elasticsearch[fmfe16][search][T#23]" "
  9243. #234 "
  9244. daemon "
  9245. prio=5 "
  9246. os_prio=15 "
  9247. tid=0x0000000b03586000 "
  9248. nid=0x198b0 "
  9249. waiting on condition "
  9250. [0x00007fffd00ff000]
  9251.  
  9252. java.lang.Thread.State: WAITING (parking)
  9253.  
  9254. at sun.misc.Unsafe.park(Native Method)
  9255.  
  9256. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9257. rrent.LinkedTransferQueue)
  9258.  
  9259. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9260. a:175)
  9261.  
  9262. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9263. ransferQueue.java:737)
  9264.  
  9265. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9266. rQueue.java:647)
  9267.  
  9268. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9269. rQueue.java:1269)
  9270.  
  9271. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9272. take(SizeBlockingQueue.java:161)
  9273.  
  9274. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9275. xecutor.java:1074)
  9276.  
  9277. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9278. lExecutor.java:1134)
  9279.  
  9280. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9281. olExecutor.java:624)
  9282.  
  9283. at java.lang.Thread.run(Thread.java:748)
  9284.  
  9285.  
  9286.  
  9287. "elasticsearch[fmfe16][search][T#22]" "
  9288. #233 "
  9289. daemon "
  9290. prio=5 "
  9291. os_prio=15 "
  9292. tid=0x0000000b03585000 "
  9293. nid=0x198ae "
  9294. waiting on condition "
  9295. [0x00007fffd0200000]
  9296.  
  9297. java.lang.Thread.State: WAITING (parking)
  9298.  
  9299. at sun.misc.Unsafe.park(Native Method)
  9300.  
  9301. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9302. rrent.LinkedTransferQueue)
  9303.  
  9304. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9305. a:175)
  9306.  
  9307. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9308. ransferQueue.java:737)
  9309.  
  9310. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9311. rQueue.java:647)
  9312.  
  9313. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9314. rQueue.java:1269)
  9315.  
  9316. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9317. take(SizeBlockingQueue.java:161)
  9318.  
  9319. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9320. xecutor.java:1074)
  9321.  
  9322. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9323. lExecutor.java:1134)
  9324.  
  9325. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9326. olExecutor.java:624)
  9327.  
  9328. at java.lang.Thread.run(Thread.java:748)
  9329.  
  9330.  
  9331.  
  9332. "elasticsearch[fmfe16][search][T#21]" "
  9333. #232 "
  9334. daemon "
  9335. prio=5 "
  9336. os_prio=15 "
  9337. tid=0x0000000b03429800 "
  9338. nid=0x198a9 "
  9339. waiting on condition "
  9340. [0x00007fffd0301000]
  9341.  
  9342. java.lang.Thread.State: WAITING (parking)
  9343.  
  9344. at sun.misc.Unsafe.park(Native Method)
  9345.  
  9346. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9347. rrent.LinkedTransferQueue)
  9348.  
  9349. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9350. a:175)
  9351.  
  9352. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9353. ransferQueue.java:737)
  9354.  
  9355. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9356. rQueue.java:647)
  9357.  
  9358. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9359. rQueue.java:1269)
  9360.  
  9361. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9362. take(SizeBlockingQueue.java:161)
  9363.  
  9364. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9365. xecutor.java:1074)
  9366.  
  9367. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9368. lExecutor.java:1134)
  9369.  
  9370. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9371. olExecutor.java:624)
  9372.  
  9373. at java.lang.Thread.run(Thread.java:748)
  9374.  
  9375.  
  9376.  
  9377. "elasticsearch[fmfe16][search][T#20]" "
  9378. #231 "
  9379. daemon "
  9380. prio=5 "
  9381. os_prio=15 "
  9382. tid=0x0000000b03573800 "
  9383. nid=0x198a6 "
  9384. waiting on condition "
  9385. [0x00007fffd0402000]
  9386.  
  9387. java.lang.Thread.State: WAITING (parking)
  9388.  
  9389. at sun.misc.Unsafe.park(Native Method)
  9390.  
  9391. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9392. rrent.LinkedTransferQueue)
  9393.  
  9394. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9395. a:175)
  9396.  
  9397. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9398. ransferQueue.java:737)
  9399.  
  9400. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9401. rQueue.java:647)
  9402.  
  9403. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9404. rQueue.java:1269)
  9405.  
  9406. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9407. take(SizeBlockingQueue.java:161)
  9408.  
  9409. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9410. xecutor.java:1074)
  9411.  
  9412. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9413. lExecutor.java:1134)
  9414.  
  9415. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9416. olExecutor.java:624)
  9417.  
  9418. at java.lang.Thread.run(Thread.java:748)
  9419.  
  9420.  
  9421.  
  9422. "elasticsearch[fmfe16][search][T#19]" "
  9423. #230 "
  9424. daemon "
  9425. prio=5 "
  9426. os_prio=15 "
  9427. tid=0x0000000b03573000 "
  9428. nid=0x1989e "
  9429. waiting on condition "
  9430. [0x00007fffd0503000]
  9431.  
  9432. java.lang.Thread.State: WAITING (parking)
  9433.  
  9434. at sun.misc.Unsafe.park(Native Method)
  9435.  
  9436. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9437. rrent.LinkedTransferQueue)
  9438.  
  9439. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9440. a:175)
  9441.  
  9442. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9443. ransferQueue.java:737)
  9444.  
  9445. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9446. rQueue.java:647)
  9447.  
  9448. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9449. rQueue.java:1269)
  9450.  
  9451. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9452. take(SizeBlockingQueue.java:161)
  9453.  
  9454. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9455. xecutor.java:1074)
  9456.  
  9457. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9458. lExecutor.java:1134)
  9459.  
  9460. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9461. olExecutor.java:624)
  9462.  
  9463. at java.lang.Thread.run(Thread.java:748)
  9464.  
  9465.  
  9466.  
  9467. "elasticsearch[fmfe16][search][T#18]" "
  9468. #229 "
  9469. daemon "
  9470. prio=5 "
  9471. os_prio=15 "
  9472. tid=0x0000000b03572000 "
  9473. nid=0x19897 "
  9474. waiting on condition "
  9475. [0x00007fffd0604000]
  9476.  
  9477. java.lang.Thread.State: WAITING (parking)
  9478.  
  9479. at sun.misc.Unsafe.park(Native Method)
  9480.  
  9481. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9482. rrent.LinkedTransferQueue)
  9483.  
  9484. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9485. a:175)
  9486.  
  9487. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9488. ransferQueue.java:737)
  9489.  
  9490. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9491. rQueue.java:647)
  9492.  
  9493. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9494. rQueue.java:1269)
  9495.  
  9496. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9497. take(SizeBlockingQueue.java:161)
  9498.  
  9499. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9500. xecutor.java:1074)
  9501.  
  9502. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9503. lExecutor.java:1134)
  9504.  
  9505. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9506. olExecutor.java:624)
  9507.  
  9508. at java.lang.Thread.run(Thread.java:748)
  9509.  
  9510.  
  9511.  
  9512. "elasticsearch[fmfe16][search][T#17]" "
  9513. #228 "
  9514. daemon "
  9515. prio=5 "
  9516. os_prio=15 "
  9517. tid=0x0000000b03571800 "
  9518. nid=0x19896 "
  9519. waiting on condition "
  9520. [0x00007fffd0705000]
  9521.  
  9522. java.lang.Thread.State: WAITING (parking)
  9523.  
  9524. at sun.misc.Unsafe.park(Native Method)
  9525.  
  9526. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9527. rrent.LinkedTransferQueue)
  9528.  
  9529. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9530. a:175)
  9531.  
  9532. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9533. ransferQueue.java:737)
  9534.  
  9535. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9536. rQueue.java:647)
  9537.  
  9538. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9539. rQueue.java:1269)
  9540.  
  9541. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9542. take(SizeBlockingQueue.java:161)
  9543.  
  9544. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9545. xecutor.java:1074)
  9546.  
  9547. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9548. lExecutor.java:1134)
  9549.  
  9550. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9551. olExecutor.java:624)
  9552.  
  9553. at java.lang.Thread.run(Thread.java:748)
  9554.  
  9555.  
  9556.  
  9557. "elasticsearch[fmfe16][search][T#16]" "
  9558. #227 "
  9559. daemon "
  9560. prio=5 "
  9561. os_prio=15 "
  9562. tid=0x0000000b03570800 "
  9563. nid=0x19894 "
  9564. waiting on condition "
  9565. [0x00007fffd0806000]
  9566.  
  9567. java.lang.Thread.State: WAITING (parking)
  9568.  
  9569. at sun.misc.Unsafe.park(Native Method)
  9570.  
  9571. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9572. rrent.LinkedTransferQueue)
  9573.  
  9574. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9575. a:175)
  9576.  
  9577. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9578. ransferQueue.java:737)
  9579.  
  9580. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9581. rQueue.java:647)
  9582.  
  9583. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9584. rQueue.java:1269)
  9585.  
  9586. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9587. take(SizeBlockingQueue.java:161)
  9588.  
  9589. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9590. xecutor.java:1074)
  9591.  
  9592. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9593. lExecutor.java:1134)
  9594.  
  9595. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9596. olExecutor.java:624)
  9597.  
  9598. at java.lang.Thread.run(Thread.java:748)
  9599.  
  9600.  
  9601.  
  9602. "elasticsearch[fmfe16][search][T#15]" "
  9603. #226 "
  9604. daemon "
  9605. prio=5 "
  9606. os_prio=15 "
  9607. tid=0x0000000b03570000 "
  9608. nid=0x1988e "
  9609. waiting on condition "
  9610. [0x00007fffd0907000]
  9611.  
  9612. java.lang.Thread.State: WAITING (parking)
  9613.  
  9614. at sun.misc.Unsafe.park(Native Method)
  9615.  
  9616. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9617. rrent.LinkedTransferQueue)
  9618.  
  9619. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9620. a:175)
  9621.  
  9622. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9623. ransferQueue.java:737)
  9624.  
  9625. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9626. rQueue.java:647)
  9627.  
  9628. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9629. rQueue.java:1269)
  9630.  
  9631. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9632. take(SizeBlockingQueue.java:161)
  9633.  
  9634. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9635. xecutor.java:1074)
  9636.  
  9637. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9638. lExecutor.java:1134)
  9639.  
  9640. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9641. olExecutor.java:624)
  9642.  
  9643. at java.lang.Thread.run(Thread.java:748)
  9644.  
  9645.  
  9646.  
  9647. "elasticsearch[fmfe16][search][T#14]" "
  9648. #225 "
  9649. daemon "
  9650. prio=5 "
  9651. os_prio=15 "
  9652. tid=0x0000000b0356f000 "
  9653. nid=0x1987c "
  9654. waiting on condition "
  9655. [0x00007fffd0a08000]
  9656.  
  9657. java.lang.Thread.State: WAITING (parking)
  9658.  
  9659. at sun.misc.Unsafe.park(Native Method)
  9660.  
  9661. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9662. rrent.LinkedTransferQueue)
  9663.  
  9664. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9665. a:175)
  9666.  
  9667. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9668. ransferQueue.java:737)
  9669.  
  9670. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9671. rQueue.java:647)
  9672.  
  9673. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9674. rQueue.java:1269)
  9675.  
  9676. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9677. take(SizeBlockingQueue.java:161)
  9678.  
  9679. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9680. xecutor.java:1074)
  9681.  
  9682. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9683. lExecutor.java:1134)
  9684.  
  9685. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9686. olExecutor.java:624)
  9687.  
  9688. at java.lang.Thread.run(Thread.java:748)
  9689.  
  9690.  
  9691.  
  9692. "elasticsearch[fmfe16][search][T#13]" "
  9693. #224 "
  9694. daemon "
  9695. prio=5 "
  9696. os_prio=15 "
  9697. tid=0x0000000b0356e800 "
  9698. nid=0x19872 "
  9699. waiting on condition "
  9700. [0x00007fffd0b09000]
  9701.  
  9702. java.lang.Thread.State: WAITING (parking)
  9703.  
  9704. at sun.misc.Unsafe.park(Native Method)
  9705.  
  9706. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9707. rrent.LinkedTransferQueue)
  9708.  
  9709. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9710. a:175)
  9711.  
  9712. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9713. ransferQueue.java:737)
  9714.  
  9715. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9716. rQueue.java:647)
  9717.  
  9718. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9719. rQueue.java:1269)
  9720.  
  9721. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9722. take(SizeBlockingQueue.java:161)
  9723.  
  9724. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9725. xecutor.java:1074)
  9726.  
  9727. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9728. lExecutor.java:1134)
  9729.  
  9730. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9731. olExecutor.java:624)
  9732.  
  9733. at java.lang.Thread.run(Thread.java:748)
  9734.  
  9735.  
  9736.  
  9737. "elasticsearch[fmfe16][search][T#12]" "
  9738. #223 "
  9739. daemon "
  9740. prio=5 "
  9741. os_prio=15 "
  9742. tid=0x0000000b0356d800 "
  9743. nid=0x19863 "
  9744. waiting on condition "
  9745. [0x00007fffd0c0a000]
  9746.  
  9747. java.lang.Thread.State: WAITING (parking)
  9748.  
  9749. at sun.misc.Unsafe.park(Native Method)
  9750.  
  9751. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9752. rrent.LinkedTransferQueue)
  9753.  
  9754. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9755. a:175)
  9756.  
  9757. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9758. ransferQueue.java:737)
  9759.  
  9760. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9761. rQueue.java:647)
  9762.  
  9763. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9764. rQueue.java:1269)
  9765.  
  9766. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9767. take(SizeBlockingQueue.java:161)
  9768.  
  9769. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9770. xecutor.java:1074)
  9771.  
  9772. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9773. lExecutor.java:1134)
  9774.  
  9775. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9776. olExecutor.java:624)
  9777.  
  9778. at java.lang.Thread.run(Thread.java:748)
  9779.  
  9780.  
  9781.  
  9782. "elasticsearch[fmfe16][search][T#11]" "
  9783. #222 "
  9784. daemon "
  9785. prio=5 "
  9786. os_prio=15 "
  9787. tid=0x0000000b0356d000 "
  9788. nid=0x1985d "
  9789. waiting on condition "
  9790. [0x00007fffd0d0b000]
  9791.  
  9792. java.lang.Thread.State: WAITING (parking)
  9793.  
  9794. at sun.misc.Unsafe.park(Native Method)
  9795.  
  9796. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9797. rrent.LinkedTransferQueue)
  9798.  
  9799. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9800. a:175)
  9801.  
  9802. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9803. ransferQueue.java:737)
  9804.  
  9805. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9806. rQueue.java:647)
  9807.  
  9808. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9809. rQueue.java:1269)
  9810.  
  9811. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9812. take(SizeBlockingQueue.java:161)
  9813.  
  9814. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9815. xecutor.java:1074)
  9816.  
  9817. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9818. lExecutor.java:1134)
  9819.  
  9820. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9821. olExecutor.java:624)
  9822.  
  9823. at java.lang.Thread.run(Thread.java:748)
  9824.  
  9825.  
  9826.  
  9827. "elasticsearch[fmfe16][search][T#10]" "
  9828. #221 "
  9829. daemon "
  9830. prio=5 "
  9831. os_prio=15 "
  9832. tid=0x0000000b0356c000 "
  9833. nid=0x1985a "
  9834. waiting on condition "
  9835. [0x00007fffd0e0c000]
  9836.  
  9837. java.lang.Thread.State: WAITING (parking)
  9838.  
  9839. at sun.misc.Unsafe.park(Native Method)
  9840.  
  9841. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9842. rrent.LinkedTransferQueue)
  9843.  
  9844. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9845. a:175)
  9846.  
  9847. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9848. ransferQueue.java:737)
  9849.  
  9850. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9851. rQueue.java:647)
  9852.  
  9853. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9854. rQueue.java:1269)
  9855.  
  9856. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9857. take(SizeBlockingQueue.java:161)
  9858.  
  9859. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9860. xecutor.java:1074)
  9861.  
  9862. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9863. lExecutor.java:1134)
  9864.  
  9865. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9866. olExecutor.java:624)
  9867.  
  9868. at java.lang.Thread.run(Thread.java:748)
  9869.  
  9870.  
  9871.  
  9872. "elasticsearch[fmfe16][search][T#9]" "
  9873. #220 "
  9874. daemon "
  9875. prio=5 "
  9876. os_prio=15 "
  9877. tid=0x0000000b0344f800 "
  9878. nid=0x19856 "
  9879. waiting on condition "
  9880. [0x00007fffd0f0d000]
  9881.  
  9882. java.lang.Thread.State: WAITING (parking)
  9883.  
  9884. at sun.misc.Unsafe.park(Native Method)
  9885.  
  9886. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9887. rrent.LinkedTransferQueue)
  9888.  
  9889. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9890. a:175)
  9891.  
  9892. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9893. ransferQueue.java:737)
  9894.  
  9895. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9896. rQueue.java:647)
  9897.  
  9898. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9899. rQueue.java:1269)
  9900.  
  9901. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9902. take(SizeBlockingQueue.java:161)
  9903.  
  9904. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9905. xecutor.java:1074)
  9906.  
  9907. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9908. lExecutor.java:1134)
  9909.  
  9910. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9911. olExecutor.java:624)
  9912.  
  9913. at java.lang.Thread.run(Thread.java:748)
  9914.  
  9915.  
  9916.  
  9917. "elasticsearch[fmfe16][search][T#8]" "
  9918. #219 "
  9919. daemon "
  9920. prio=5 "
  9921. os_prio=15 "
  9922. tid=0x0000000b0344e800 "
  9923. nid=0x1984f "
  9924. waiting on condition "
  9925. [0x00007fffd100e000]
  9926.  
  9927. java.lang.Thread.State: WAITING (parking)
  9928.  
  9929. at sun.misc.Unsafe.park(Native Method)
  9930.  
  9931. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9932. rrent.LinkedTransferQueue)
  9933.  
  9934. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9935. a:175)
  9936.  
  9937. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9938. ransferQueue.java:737)
  9939.  
  9940. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9941. rQueue.java:647)
  9942.  
  9943. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9944. rQueue.java:1269)
  9945.  
  9946. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9947. take(SizeBlockingQueue.java:161)
  9948.  
  9949. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9950. xecutor.java:1074)
  9951.  
  9952. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9953. lExecutor.java:1134)
  9954.  
  9955. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  9956. olExecutor.java:624)
  9957.  
  9958. at java.lang.Thread.run(Thread.java:748)
  9959.  
  9960.  
  9961.  
  9962. "elasticsearch[fmfe16][search][T#7]" "
  9963. #218 "
  9964. daemon "
  9965. prio=5 "
  9966. os_prio=15 "
  9967. tid=0x0000000b0344e000 "
  9968. nid=0x1984b "
  9969. waiting on condition "
  9970. [0x00007fffd110f000]
  9971.  
  9972. java.lang.Thread.State: WAITING (parking)
  9973.  
  9974. at sun.misc.Unsafe.park(Native Method)
  9975.  
  9976. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  9977. rrent.LinkedTransferQueue)
  9978.  
  9979. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  9980. a:175)
  9981.  
  9982. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  9983. ransferQueue.java:737)
  9984.  
  9985. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  9986. rQueue.java:647)
  9987.  
  9988. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  9989. rQueue.java:1269)
  9990.  
  9991. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  9992. take(SizeBlockingQueue.java:161)
  9993.  
  9994. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  9995. xecutor.java:1074)
  9996.  
  9997. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  9998. lExecutor.java:1134)
  9999.  
  10000. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10001. olExecutor.java:624)
  10002.  
  10003. at java.lang.Thread.run(Thread.java:748)
  10004.  
  10005.  
  10006.  
  10007. "elasticsearch[fmfe16][search][T#6]" "
  10008. #217 "
  10009. daemon "
  10010. prio=5 "
  10011. os_prio=15 "
  10012. tid=0x0000000b0344d000 "
  10013. nid=0x19839 "
  10014. waiting on condition "
  10015. [0x00007fffd1210000]
  10016.  
  10017. java.lang.Thread.State: WAITING (parking)
  10018.  
  10019. at sun.misc.Unsafe.park(Native Method)
  10020.  
  10021. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  10022. rrent.LinkedTransferQueue)
  10023.  
  10024. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10025. a:175)
  10026.  
  10027. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  10028. ransferQueue.java:737)
  10029.  
  10030. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  10031. rQueue.java:647)
  10032.  
  10033. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  10034. rQueue.java:1269)
  10035.  
  10036. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  10037. take(SizeBlockingQueue.java:161)
  10038.  
  10039. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10040. xecutor.java:1074)
  10041.  
  10042. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10043. lExecutor.java:1134)
  10044.  
  10045. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10046. olExecutor.java:624)
  10047.  
  10048. at java.lang.Thread.run(Thread.java:748)
  10049.  
  10050.  
  10051.  
  10052. "elasticsearch[fmfe16][search][T#5]" "
  10053. #216 "
  10054. daemon "
  10055. prio=5 "
  10056. os_prio=15 "
  10057. tid=0x0000000b0344c800 "
  10058. nid=0x19838 "
  10059. waiting on condition "
  10060. [0x00007fffd1311000]
  10061.  
  10062. java.lang.Thread.State: WAITING (parking)
  10063.  
  10064. at sun.misc.Unsafe.park(Native Method)
  10065.  
  10066. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  10067. rrent.LinkedTransferQueue)
  10068.  
  10069. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10070. a:175)
  10071.  
  10072. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  10073. ransferQueue.java:737)
  10074.  
  10075. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  10076. rQueue.java:647)
  10077.  
  10078. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  10079. rQueue.java:1269)
  10080.  
  10081. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  10082. take(SizeBlockingQueue.java:161)
  10083.  
  10084. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10085. xecutor.java:1074)
  10086.  
  10087. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10088. lExecutor.java:1134)
  10089.  
  10090. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10091. olExecutor.java:624)
  10092.  
  10093. at java.lang.Thread.run(Thread.java:748)
  10094.  
  10095.  
  10096.  
  10097. "elasticsearch[fmfe16][search][T#4]" "
  10098. #215 "
  10099. daemon "
  10100. prio=5 "
  10101. os_prio=15 "
  10102. tid=0x0000000b0344b800 "
  10103. nid=0x19833 "
  10104. waiting on condition "
  10105. [0x00007fffd1412000]
  10106.  
  10107. java.lang.Thread.State: WAITING (parking)
  10108.  
  10109. at sun.misc.Unsafe.park(Native Method)
  10110.  
  10111. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  10112. rrent.LinkedTransferQueue)
  10113.  
  10114. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10115. a:175)
  10116.  
  10117. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  10118. ransferQueue.java:737)
  10119.  
  10120. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  10121. rQueue.java:647)
  10122.  
  10123. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  10124. rQueue.java:1269)
  10125.  
  10126. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  10127. take(SizeBlockingQueue.java:161)
  10128.  
  10129. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10130. xecutor.java:1074)
  10131.  
  10132. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10133. lExecutor.java:1134)
  10134.  
  10135. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10136. olExecutor.java:624)
  10137.  
  10138. at java.lang.Thread.run(Thread.java:748)
  10139.  
  10140.  
  10141.  
  10142. "elasticsearch[fmfe16][search][T#3]" "
  10143. #214 "
  10144. daemon "
  10145. prio=5 "
  10146. os_prio=15 "
  10147. tid=0x0000000b0344b000 "
  10148. nid=0x19830 "
  10149. waiting on condition "
  10150. [0x00007fffd1513000]
  10151.  
  10152. java.lang.Thread.State: WAITING (parking)
  10153.  
  10154. at sun.misc.Unsafe.park(Native Method)
  10155.  
  10156. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  10157. rrent.LinkedTransferQueue)
  10158.  
  10159. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10160. a:175)
  10161.  
  10162. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  10163. ransferQueue.java:737)
  10164.  
  10165. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  10166. rQueue.java:647)
  10167.  
  10168. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  10169. rQueue.java:1269)
  10170.  
  10171. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  10172. take(SizeBlockingQueue.java:161)
  10173.  
  10174. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10175. xecutor.java:1074)
  10176.  
  10177. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10178. lExecutor.java:1134)
  10179.  
  10180. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10181. olExecutor.java:624)
  10182.  
  10183. at java.lang.Thread.run(Thread.java:748)
  10184.  
  10185.  
  10186.  
  10187. "elasticsearch[fmfe16][search][T#2]" "
  10188. #213 "
  10189. daemon "
  10190. prio=5 "
  10191. os_prio=15 "
  10192. tid=0x0000000b0344a000 "
  10193. nid=0x1982c "
  10194. waiting on condition "
  10195. [0x00007fffd1614000]
  10196.  
  10197. java.lang.Thread.State: WAITING (parking)
  10198.  
  10199. at sun.misc.Unsafe.park(Native Method)
  10200.  
  10201. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  10202. rrent.LinkedTransferQueue)
  10203.  
  10204. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10205. a:175)
  10206.  
  10207. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  10208. ransferQueue.java:737)
  10209.  
  10210. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  10211. rQueue.java:647)
  10212.  
  10213. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  10214. rQueue.java:1269)
  10215.  
  10216. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  10217. take(SizeBlockingQueue.java:161)
  10218.  
  10219. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10220. xecutor.java:1074)
  10221.  
  10222. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10223. lExecutor.java:1134)
  10224.  
  10225. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10226. olExecutor.java:624)
  10227.  
  10228. at java.lang.Thread.run(Thread.java:748)
  10229.  
  10230.  
  10231.  
  10232. "elasticsearch[fmfe16][search][T#1]" "
  10233. #212 "
  10234. daemon "
  10235. prio=5 "
  10236. os_prio=15 "
  10237. tid=0x0000000b0342a800 "
  10238. nid=0x19827 "
  10239. waiting on condition "
  10240. [0x00007fffd1715000]
  10241.  
  10242. java.lang.Thread.State: WAITING (parking)
  10243.  
  10244. at sun.misc.Unsafe.park(Native Method)
  10245.  
  10246. - parking to wait for <0x0000000873186c58> (a java.util.concu\
  10247. rrent.LinkedTransferQueue)
  10248.  
  10249. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10250. a:175)
  10251.  
  10252. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  10253. ransferQueue.java:737)
  10254.  
  10255. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  10256. rQueue.java:647)
  10257.  
  10258. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  10259. rQueue.java:1269)
  10260.  
  10261. at org.elasticsearch.common.util.concurrent.SizeBlockingQueue.\
  10262. take(SizeBlockingQueue.java:161)
  10263.  
  10264. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10265. xecutor.java:1074)
  10266.  
  10267. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10268. lExecutor.java:1134)
  10269.  
  10270. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10271. olExecutor.java:624)
  10272.  
  10273. at java.lang.Thread.run(Thread.java:748)
  10274.  
  10275.  
  10276.  
  10277. "DestroyJavaVM" "
  10278. #210 "
  10279. prio=5 "
  10280. os_prio=15 "
  10281. tid=0x0000000803085800 "
  10282. nid=0x187f9 "
  10283. waiting on condition "
  10284. [0x0000000000000000]
  10285.  
  10286. java.lang.Thread.State: RUNNABLE
  10287.  
  10288.  
  10289.  
  10290. "elasticsearch[fmfe16][management][T#1]" "
  10291. #159 "
  10292. daemon "
  10293. prio=5 "
  10294. os_prio=15 "
  10295. tid=0x0000000b04c88800 "
  10296. nid=0x1966f "
  10297. waiting on condition "
  10298. [0x00007fffd4c4a000]
  10299.  
  10300. java.lang.Thread.State: WAITING (parking)
  10301.  
  10302. at sun.misc.Unsafe.park(Native Method)
  10303.  
  10304. - parking to wait for <0x000000089330a078> (a org.elasticsear\
  10305. ch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
  10306.  
  10307. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10308. a:175)
  10309.  
  10310. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  10311. ransferQueue.java:737)
  10312.  
  10313. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  10314. rQueue.java:647)
  10315.  
  10316. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  10317. rQueue.java:1269)
  10318.  
  10319. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10320. xecutor.java:1074)
  10321.  
  10322. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10323. lExecutor.java:1134)
  10324.  
  10325. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10326. olExecutor.java:624)
  10327.  
  10328. at java.lang.Thread.run(Thread.java:748)
  10329.  
  10330.  
  10331.  
  10332. "elasticsearch[fmfe16][fetch_shard_started][T#47]" "
  10333. #124 "
  10334. daemon "
  10335. prio=5 "
  10336. os_prio=15 "
  10337. tid=0x0000000b03dfd000 "
  10338. nid=0x19527 "
  10339. waiting on condition "
  10340. [0x00007fffd6f6d000]
  10341.  
  10342. java.lang.Thread.State: WAITING (parking)
  10343.  
  10344. at sun.misc.Unsafe.park(Native Method)
  10345.  
  10346. - parking to wait for <0x0000000872bd6160> (a org.elasticsear\
  10347. ch.common.util.concurrent.EsExecutors$ExecutorScalingQueue)
  10348.  
  10349. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10350. a:175)
  10351.  
  10352. at java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedT\
  10353. ransferQueue.java:737)
  10354.  
  10355. at java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransfe\
  10356. rQueue.java:647)
  10357.  
  10358. at java.util.concurrent.LinkedTransferQueue.take(LinkedTransfe\
  10359. rQueue.java:1269)
  10360.  
  10361. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10362. xecutor.java:1074)
  10363.  
  10364. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10365. lExecutor.java:1134)
  10366.  
  10367. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10368. olExecutor.java:624)
  10369.  
  10370. at java.lang.Thread.run(Thread.java:748)
  10371.  
  10372.  
  10373.  
  10374. "elasticsearch[fmfe16][scheduler][T#1]" "
  10375. #24 "
  10376. daemon "
  10377. prio=5 "
  10378. os_prio=15 "
  10379. tid=0x0000000afcc7a000 "
  10380. nid=0x188be "
  10381. waiting on condition "
  10382. [0x00007fffdd2d0000]
  10383.  
  10384. java.lang.Thread.State: TIMED_WAITING (parking)
  10385.  
  10386. at sun.misc.Unsafe.park(Native Method)
  10387.  
  10388. - parking to wait for <0x0000000872bcddd8> (a java.util.concu\
  10389. rrent.locks.AbstractQueuedSynchronizer$ConditionObject)
  10390.  
  10391. at java.util.concurrent.locks.LockSupport.parkNanos(LockSuppor\
  10392. t.java:215)
  10393.  
  10394. at java.util.concurrent.locks.AbstractQueuedSynchronizer$Condi\
  10395. tionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
  10396.  
  10397. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWor\
  10398. kQueue.take(ScheduledThreadPoolExecutor.java:1093)
  10399.  
  10400. at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWor\
  10401. kQueue.take(ScheduledThreadPoolExecutor.java:809)
  10402.  
  10403. at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolE\
  10404. xecutor.java:1074)
  10405.  
  10406. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  10407. lExecutor.java:1134)
  10408.  
  10409. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  10410. olExecutor.java:624)
  10411.  
  10412. at java.lang.Thread.run(Thread.java:748)
  10413.  
  10414.  
  10415.  
  10416. "elasticsearch[fmfe16][[timer]]" "
  10417. #23 "
  10418. daemon "
  10419. prio=5 "
  10420. os_prio=15 "
  10421. tid=0x0000000af9b22000 "
  10422. nid=0x190be "
  10423. sleeping"
  10424. [0x00007fffdd3d1000]
  10425.  
  10426. java.lang.Thread.State: TIMED_WAITING (sleeping)
  10427.  
  10428. at java.lang.Thread.sleep(Native Method)
  10429.  
  10430. at org.elasticsearch.threadpool.ThreadPool$CachedTimeThread.ru\
  10431. n(ThreadPool.java:541)
  10432.  
  10433.  
  10434.  
  10435. "Service Thread" "
  10436. #18 "
  10437. daemon "
  10438. prio=9 "
  10439. os_prio=15 "
  10440. tid=0x0000000a975dd000 "
  10441. nid=0x190b0 "
  10442. runnable "
  10443. [0x0000000000000000]
  10444.  
  10445. java.lang.Thread.State: RUNNABLE
  10446.  
  10447.  
  10448.  
  10449. "C1 CompilerThread11" "
  10450. #17 "
  10451. daemon "
  10452. prio=9 "
  10453. os_prio=15 "
  10454. tid=0x0000000a975c7800 "
  10455. nid=0x190a0 "
  10456. waiting on condition "
  10457. [0x0000000000000000]
  10458.  
  10459. java.lang.Thread.State: RUNNABLE
  10460.  
  10461.  
  10462.  
  10463. "C1 CompilerThread10" "
  10464. #16 "
  10465. daemon "
  10466. prio=9 "
  10467. os_prio=15 "
  10468. tid=0x0000000a975c6800 "
  10469. nid=0x1909f "
  10470. waiting on condition "
  10471. [0x0000000000000000]
  10472.  
  10473. java.lang.Thread.State: RUNNABLE
  10474.  
  10475.  
  10476.  
  10477. "C1 CompilerThread9" "
  10478. #15 "
  10479. daemon "
  10480. prio=9 "
  10481. os_prio=15 "
  10482. tid=0x0000000a975c6000 "
  10483. nid=0x1909a "
  10484. waiting on condition "
  10485. [0x0000000000000000]
  10486.  
  10487. java.lang.Thread.State: RUNNABLE
  10488.  
  10489.  
  10490.  
  10491. "C1 CompilerThread8" "
  10492. #14 "
  10493. daemon "
  10494. prio=9 "
  10495. os_prio=15 "
  10496. tid=0x0000000a975c5000 "
  10497. nid=0x19097 "
  10498. waiting on condition "
  10499. [0x0000000000000000]
  10500.  
  10501. java.lang.Thread.State: RUNNABLE
  10502.  
  10503.  
  10504.  
  10505. "C2 CompilerThread7" "
  10506. #13 "
  10507. daemon "
  10508. prio=9 "
  10509. os_prio=15 "
  10510. tid=0x0000000a975c4800 "
  10511. nid=0x19093 "
  10512. waiting on condition "
  10513. [0x0000000000000000]
  10514.  
  10515. java.lang.Thread.State: RUNNABLE
  10516.  
  10517.  
  10518.  
  10519. "C2 CompilerThread6" "
  10520. #12 "
  10521. daemon "
  10522. prio=9 "
  10523. os_prio=15 "
  10524. tid=0x0000000a975c3800 "
  10525. nid=0x1908f "
  10526. waiting on condition "
  10527. [0x0000000000000000]
  10528.  
  10529. java.lang.Thread.State: RUNNABLE
  10530.  
  10531.  
  10532.  
  10533. "C2 CompilerThread5" "
  10534. #11 "
  10535. daemon "
  10536. prio=9 "
  10537. os_prio=15 "
  10538. tid=0x0000000a975c3000 "
  10539. nid=0x19083 "
  10540. waiting on condition "
  10541. [0x0000000000000000]
  10542.  
  10543. java.lang.Thread.State: RUNNABLE
  10544.  
  10545.  
  10546.  
  10547. "C2 CompilerThread4" "
  10548. #10 "
  10549. daemon "
  10550. prio=9 "
  10551. os_prio=15 "
  10552. tid=0x0000000a975c2000 "
  10553. nid=0x1907f "
  10554. waiting on condition "
  10555. [0x0000000000000000]
  10556.  
  10557. java.lang.Thread.State: RUNNABLE
  10558.  
  10559.  
  10560.  
  10561. "C2 CompilerThread3" "
  10562. #9 "
  10563. daemon "
  10564. prio=9 "
  10565. os_prio=15 "
  10566. tid=0x000000080308b800 "
  10567. nid=0x1907c "
  10568. waiting on condition "
  10569. [0x0000000000000000]
  10570.  
  10571. java.lang.Thread.State: RUNNABLE
  10572.  
  10573.  
  10574.  
  10575. "C2 CompilerThread2" "
  10576. #8 "
  10577. daemon "
  10578. prio=9 "
  10579. os_prio=15 "
  10580. tid=0x000000080308a800 "
  10581. nid=0x19076 "
  10582. waiting on condition "
  10583. [0x0000000000000000]
  10584.  
  10585. java.lang.Thread.State: RUNNABLE
  10586.  
  10587.  
  10588.  
  10589. "C2 CompilerThread1" "
  10590. #7 "
  10591. daemon "
  10592. prio=9 "
  10593. os_prio=15 "
  10594. tid=0x000000080308a000 "
  10595. nid=0x1906c "
  10596. waiting on condition "
  10597. [0x0000000000000000]
  10598.  
  10599. java.lang.Thread.State: RUNNABLE
  10600.  
  10601.  
  10602.  
  10603. "C2 CompilerThread0" "
  10604. #6 "
  10605. daemon "
  10606. prio=9 "
  10607. os_prio=15 "
  10608. tid=0x0000000803089000 "
  10609. nid=0x19022 "
  10610. waiting on condition "
  10611. [0x0000000000000000]
  10612.  
  10613. java.lang.Thread.State: RUNNABLE
  10614.  
  10615.  
  10616.  
  10617. "Signal Dispatcher" "
  10618. #5 "
  10619. daemon "
  10620. prio=9 "
  10621. os_prio=15 "
  10622. tid=0x0000000803088800 "
  10623. nid=0x19002 "
  10624. waiting on condition "
  10625. [0x0000000000000000]
  10626.  
  10627. java.lang.Thread.State: RUNNABLE
  10628.  
  10629.  
  10630.  
  10631. "Surrogate Locker Thread (Concurrent GC)" "
  10632. #4 "
  10633. daemon "
  10634. prio=9 "
  10635. os_prio=15 "
  10636. tid=0x0000000a973e2000 "
  10637. nid=0x18fcb "
  10638. waiting on condition "
  10639. [0x0000000000000000]
  10640.  
  10641. java.lang.Thread.State: RUNNABLE
  10642.  
  10643.  
  10644.  
  10645. "Finalizer" "
  10646. #3 "
  10647. daemon "
  10648. prio=8 "
  10649. os_prio=15 "
  10650. tid=0x0000000803087800 "
  10651. nid=0x18fb6 "
  10652. in Object.wait() "
  10653. [0x00007fffde4e2000]
  10654.  
  10655. java.lang.Thread.State: WAITING (on object monitor)
  10656.  
  10657. at java.lang.Object.wait(Native Method)
  10658.  
  10659. at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143\
  10660. )
  10661.  
  10662. - locked <0x0000000872be89f8> "
  10663. (a java.lang.ref.ReferenceQueue$Lock)
  10664.  
  10665. at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164\
  10666. )
  10667.  
  10668. at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:\
  10669. 209)
  10670.  
  10671.  
  10672.  
  10673. "Reference Handler" "
  10674. #2 "
  10675. daemon "
  10676. prio=10 "
  10677. os_prio=15 "
  10678. tid=0x0000000803087000 "
  10679. nid=0x18f37 "
  10680. in Object.wait() "
  10681. [0x00007fffde5e3000]
  10682.  
  10683. java.lang.Thread.State: WAITING (on object monitor)
  10684.  
  10685. at java.lang.Object.wait(Native Method)
  10686.  
  10687. at java.lang.Object.wait(Object.java:502)
  10688.  
  10689. at java.lang.ref.Reference.tryHandlePending(Reference.java:191\
  10690. )
  10691.  
  10692. - locked <0x0000000872be8a28> "
  10693. (a java.lang.ref.Reference$Lock)
  10694.  
  10695. at java.lang.ref.Reference$ReferenceHandler.run(Reference.java\
  10696. :153)
  10697.  
  10698.  
  10699.  
  10700. "VM Thread" "
  10701. os_prio=15 "
  10702. tid=0x0000000a96f2a000 "
  10703. nid=0x18f31 "
  10704. runnable "
  10705.  
  10706.  
  10707.  
  10708.  
  10709. "Gang worker#0 (Parallel GC Threads)" "
  10710. os_prio=15 "
  10711. tid=0x00000008030cc000 "
  10712. nid=0x1884d "
  10713. runnable "
  10714.  
  10715.  
  10716.  
  10717.  
  10718. "Gang worker#1 (Parallel GC Threads)" "
  10719. os_prio=15 "
  10720. tid=0x00000008030cd000 "
  10721. nid=0x18874 "
  10722. runnable "
  10723.  
  10724.  
  10725.  
  10726.  
  10727. "Gang worker#2 (Parallel GC Threads)" "
  10728. os_prio=15 "
  10729. tid=0x00000008030cd800 "
  10730. nid=0x1888f "
  10731. runnable "
  10732.  
  10733.  
  10734.  
  10735.  
  10736. "Gang worker#3 (Parallel GC Threads)" "
  10737. os_prio=15 "
  10738. tid=0x00000008030ce000 "
  10739. nid=0x188ee "
  10740. runnable "
  10741.  
  10742.  
  10743.  
  10744.  
  10745. "Gang worker#4 (Parallel GC Threads)" "
  10746. os_prio=15 "
  10747. tid=0x00000008030ce800 "
  10748. nid=0x18925 "
  10749. runnable "
  10750.  
  10751.  
  10752.  
  10753.  
  10754. "Gang worker#5 (Parallel GC Threads)" "
  10755. os_prio=15 "
  10756. tid=0x00000008030cf800 "
  10757. nid=0x189bc "
  10758. runnable "
  10759.  
  10760.  
  10761.  
  10762.  
  10763. "Gang worker#6 (Parallel GC Threads)" "
  10764. os_prio=15 "
  10765. tid=0x00000008030d0000 "
  10766. nid=0x18b1f "
  10767. runnable "
  10768.  
  10769.  
  10770.  
  10771.  
  10772. "Gang worker#7 (Parallel GC Threads)" "
  10773. os_prio=15 "
  10774. tid=0x00000008030d0800 "
  10775. nid=0x18b4a "
  10776. runnable "
  10777.  
  10778.  
  10779.  
  10780.  
  10781. "Gang worker#8 (Parallel GC Threads)" "
  10782. os_prio=15 "
  10783. tid=0x00000008030d1000 "
  10784. nid=0x18b54 "
  10785. runnable "
  10786.  
  10787.  
  10788.  
  10789.  
  10790. "Gang worker#9 (Parallel GC Threads)" "
  10791. os_prio=15 "
  10792. tid=0x00000008030d2000 "
  10793. nid=0x18b55 "
  10794. runnable "
  10795.  
  10796.  
  10797.  
  10798.  
  10799. "Gang worker#10 (Parallel GC Threads)" "
  10800. os_prio=15 "
  10801. tid=0x00000008030d2800 "
  10802. nid=0x18b89 "
  10803. runnable "
  10804.  
  10805.  
  10806.  
  10807.  
  10808. "Gang worker#11 (Parallel GC Threads)" "
  10809. os_prio=15 "
  10810. tid=0x00000008030d3000 "
  10811. nid=0x18bac "
  10812. runnable "
  10813.  
  10814.  
  10815.  
  10816.  
  10817. "Gang worker#12 (Parallel GC Threads)" "
  10818. os_prio=15 "
  10819. tid=0x00000008030d3800 "
  10820. nid=0x18bb2 "
  10821. runnable "
  10822.  
  10823.  
  10824.  
  10825.  
  10826. "Gang worker#13 (Parallel GC Threads)" "
  10827. os_prio=15 "
  10828. tid=0x00000008030d4800 "
  10829. nid=0x18bb6 "
  10830. runnable "
  10831.  
  10832.  
  10833.  
  10834.  
  10835. "Gang worker#14 (Parallel GC Threads)" "
  10836. os_prio=15 "
  10837. tid=0x00000008030d5000 "
  10838. nid=0x18bc1 "
  10839. runnable "
  10840.  
  10841.  
  10842.  
  10843.  
  10844. "Gang worker#15 (Parallel GC Threads)" "
  10845. os_prio=15 "
  10846. tid=0x00000008030d5800 "
  10847. nid=0x18c12 "
  10848. runnable "
  10849.  
  10850.  
  10851.  
  10852.  
  10853. "Gang worker#16 (Parallel GC Threads)" "
  10854. os_prio=15 "
  10855. tid=0x00000008030db000 "
  10856. nid=0x18c78 "
  10857. runnable "
  10858.  
  10859.  
  10860.  
  10861.  
  10862. "Gang worker#17 (Parallel GC Threads)" "
  10863. os_prio=15 "
  10864. tid=0x00000008030dc000 "
  10865. nid=0x18cd2 "
  10866. runnable "
  10867.  
  10868.  
  10869.  
  10870.  
  10871. "Concurrent Mark-Sweep GC Thread" "
  10872. os_prio=15 "
  10873. tid=0x00000008030df800 "
  10874. nid=0x18ed1 "
  10875. runnable "
  10876.  
  10877.  
  10878.  
  10879.  
  10880. "Gang worker#0 (Parallel CMS Threads)" "
  10881. os_prio=15 "
  10882. tid=0x00000008030dc800 "
  10883. nid=0x188f5 "
  10884. runnable "
  10885.  
  10886.  
  10887.  
  10888.  
  10889. "Gang worker#1 (Parallel CMS Threads)" "
  10890. os_prio=15 "
  10891. tid=0x00000008030dd000 "
  10892. nid=0x18b2a "
  10893. runnable "
  10894.  
  10895.  
  10896.  
  10897.  
  10898. "Gang worker#2 (Parallel CMS Threads)" "
  10899. os_prio=15 "
  10900. tid=0x00000008030dd800 "
  10901. nid=0x18d87 "
  10902. runnable "
  10903.  
  10904.  
  10905.  
  10906.  
  10907. "Gang worker#3 (Parallel CMS Threads)" "
  10908. os_prio=15 "
  10909. tid=0x00000008030de800 "
  10910. nid=0x18e86 "
  10911. runnable "
  10912.  
  10913.  
  10914.  
  10915.  
  10916. "Gang worker#4 (Parallel CMS Threads)" "
  10917. os_prio=15 "
  10918. tid=0x00000008030df000 "
  10919. nid=0x18ebc "
  10920. runnable "
  10921.  
  10922.  
  10923.  
  10924.  
  10925. "VM Periodic Task Thread" "
  10926. os_prio=15 "
  10927. tid=0x0000000a96f2a800 "
  10928. nid=0x190b1 "
  10929. waiting on condition "
  10930.  
  10931.  
  10932.  
  10933.  
  10934. JNI global references: 8029
  10935.  
  10936.  
  10937.  
  10938.  
  10939.  
  10940. Found one Java-level deadlock:
  10941.  
  10942. ============================="
  10943.  
  10944.  
  10945. "elasticsearch[fmfe16][index][T#24]":
  10946.  
  10947. waiting for ownable synchronizer 0x00000009df4b02c0, (a java.util.co\
  10948. ncurrent.locks.ReentrantLock$NonfairSync)"
  10949. ,
  10950. which is held by "elasticsearch[fmfe16][index][T#9]\
  10951. ""
  10952.  
  10953.  
  10954. "elasticsearch[fmfe16][index][T#9]":
  10955.  
  10956. waiting to lock monitor 0x0000000afc30c4e8"
  10957. (object 0x00000008b40309c8, a org.elasticsearch.index.engine.LiveVers\
  10958. ionMap)"
  10959. ,
  10960. which is held by "elasticsearch[fmfe16][get][T#14]""
  10961.  
  10962.  
  10963. "elasticsearch[fmfe16][get][T#14]":
  10964.  
  10965. waiting for ownable synchronizer 0x00000009df4b02c0, (a java.util.co\
  10966. ncurrent.locks.ReentrantLock$NonfairSync)"
  10967. ,
  10968. which is held by "elasticsearch[fmfe16][index][T#9]\
  10969. ""
  10970.  
  10971.  
  10972.  
  10973.  
  10974. Java stack information for the threads listed above:
  10975.  
  10976. ===================================================
  10977.  
  10978. "elasticsearch[fmfe16][index][T#24]":
  10979.  
  10980. at sun.misc.Unsafe.park(Native Method)
  10981.  
  10982. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  10983. rrent.locks.ReentrantLock$NonfairSync)
  10984.  
  10985. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  10986. a:175)
  10987.  
  10988. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  10989. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  10990.  
  10991. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  10992. reQueued(AbstractQueuedSynchronizer.java:870)
  10993.  
  10994. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  10995. re(AbstractQueuedSynchronizer.java:1199)
  10996.  
  10997. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  10998. eentrantLock.java:209)
  10999.  
  11000. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  11001. .java:285)
  11002.  
  11003. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  11004. KeyedLock.java:76)
  11005.  
  11006. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  11007. iveVersionMap.java:431)
  11008.  
  11009. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  11010. ombstones(InternalEngine.java:1661)
  11011.  
  11012. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  11013. etedTombstones(InternalEngine.java:1348)
  11014.  
  11015. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  11016. nalEngine.java:1427)
  11017.  
  11018. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  11019. ngine.java:637)
  11020.  
  11021. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  11022. a:850)
  11023.  
  11024. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  11025. etService.java:153)
  11026.  
  11027. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  11028. vice.java:81)
  11029.  
  11030. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  11031. Helper.java:74)
  11032.  
  11033. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  11034. Operation(TransportUpdateAction.java:174)
  11035.  
  11036. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  11037. Operation(TransportUpdateAction.java:167)
  11038.  
  11039. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  11040. Operation(TransportUpdateAction.java:67)
  11041.  
  11042. at org.elasticsearch.action.support.single.instance.TransportI\
  11043. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  11044. nsportInstanceSingleOperationAction.java:243)
  11045.  
  11046. at org.elasticsearch.action.support.single.instance.TransportI\
  11047. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  11048. nsportInstanceSingleOperationAction.java:239)
  11049.  
  11050. at org.elasticsearch.transport.TransportRequestHandler.message\
  11051. Received(TransportRequestHandler.java:30)
  11052.  
  11053. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  11054. essageReceived(RequestHandlerRegistry.java:66)
  11055.  
  11056. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  11057. un(TcpTransport.java:1555)
  11058.  
  11059. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  11060. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  11061.  
  11062. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  11063. un(AbstractRunnable.java:37)
  11064.  
  11065. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  11066. lExecutor.java:1149)
  11067.  
  11068. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  11069. olExecutor.java:624)
  11070.  
  11071. at java.lang.Thread.run(Thread.java:748)
  11072.  
  11073. "elasticsearch[fmfe16][index][T#9]":
  11074.  
  11075. at org.elasticsearch.index.engine.InternalEngine.getVersionFro\
  11076. mMap(InternalEngine.java:732)
  11077.  
  11078. - waiting to lock <0x00000008b40309c8> "
  11079. (a org.elasticsearch.index.engine.LiveVersionMap)
  11080.  
  11081. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  11082. ngine.java:627)
  11083.  
  11084. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  11085. a:850)
  11086.  
  11087. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  11088. etService.java:153)
  11089.  
  11090. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  11091. vice.java:81)
  11092.  
  11093. at org.elasticsearch.action.update.UpdateHelper.prepare(Update\
  11094. Helper.java:74)
  11095.  
  11096. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  11097. Operation(TransportUpdateAction.java:174)
  11098.  
  11099. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  11100. Operation(TransportUpdateAction.java:167)
  11101.  
  11102. at org.elasticsearch.action.update.TransportUpdateAction.shard\
  11103. Operation(TransportUpdateAction.java:67)
  11104.  
  11105. at org.elasticsearch.action.support.single.instance.TransportI\
  11106. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  11107. nsportInstanceSingleOperationAction.java:243)
  11108.  
  11109. at org.elasticsearch.action.support.single.instance.TransportI\
  11110. nstanceSingleOperationAction$ShardTransportHandler.messageReceived(Tra\
  11111. nsportInstanceSingleOperationAction.java:239)
  11112.  
  11113. at org.elasticsearch.transport.TransportRequestHandler.message\
  11114. Received(TransportRequestHandler.java:30)
  11115.  
  11116. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  11117. essageReceived(RequestHandlerRegistry.java:66)
  11118.  
  11119. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  11120. un(TcpTransport.java:1555)
  11121.  
  11122. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  11123. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  11124.  
  11125. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  11126. un(AbstractRunnable.java:37)
  11127.  
  11128. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  11129. lExecutor.java:1149)
  11130.  
  11131. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  11132. olExecutor.java:624)
  11133.  
  11134. at java.lang.Thread.run(Thread.java:748)
  11135.  
  11136. "elasticsearch[fmfe16][get][T#14]":
  11137.  
  11138. at sun.misc.Unsafe.park(Native Method)
  11139.  
  11140. - parking to wait for <0x00000009df4b02c0> (a java.util.concu\
  11141. rrent.locks.ReentrantLock$NonfairSync)
  11142.  
  11143. at java.util.concurrent.locks.LockSupport.park(LockSupport.jav\
  11144. a:175)
  11145.  
  11146. at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkA\
  11147. ndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
  11148.  
  11149. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  11150. reQueued(AbstractQueuedSynchronizer.java:870)
  11151.  
  11152. at java.util.concurrent.locks.AbstractQueuedSynchronizer.acqui\
  11153. re(AbstractQueuedSynchronizer.java:1199)
  11154.  
  11155. at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(R\
  11156. eentrantLock.java:209)
  11157.  
  11158. at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock\
  11159. .java:285)
  11160.  
  11161. at org.elasticsearch.common.util.concurrent.KeyedLock.acquire(\
  11162. KeyedLock.java:76)
  11163.  
  11164. at org.elasticsearch.index.engine.LiveVersionMap.acquireLock(L\
  11165. iveVersionMap.java:431)
  11166.  
  11167. at org.elasticsearch.index.engine.InternalEngine.pruneDeletedT\
  11168. ombstones(InternalEngine.java:1661)
  11169.  
  11170. at org.elasticsearch.index.engine.InternalEngine.maybePruneDel\
  11171. etedTombstones(InternalEngine.java:1348)
  11172.  
  11173. at org.elasticsearch.index.engine.InternalEngine.refresh(Inter\
  11174. nalEngine.java:1427)
  11175.  
  11176. at org.elasticsearch.index.engine.InternalEngine.getVersionFro\
  11177. mMap(InternalEngine.java:737)
  11178.  
  11179. - locked <0x00000008b40309c8> "
  11180. (a org.elasticsearch.index.engine.LiveVersionMap)
  11181.  
  11182. at org.elasticsearch.index.engine.InternalEngine.get(InternalE\
  11183. ngine.java:627)
  11184.  
  11185. at org.elasticsearch.index.shard.IndexShard.get(IndexShard.jav\
  11186. a:850)
  11187.  
  11188. at org.elasticsearch.index.get.ShardGetService.innerGet(ShardG\
  11189. etService.java:153)
  11190.  
  11191. at org.elasticsearch.index.get.ShardGetService.get(ShardGetSer\
  11192. vice.java:81)
  11193.  
  11194. at org.elasticsearch.action.get.TransportGetAction.shardOperat\
  11195. ion(TransportGetAction.java:88)
  11196.  
  11197. at org.elasticsearch.action.get.TransportGetAction.shardOperat\
  11198. ion(TransportGetAction.java:44)
  11199.  
  11200. at org.elasticsearch.action.support.single.shard.TransportSing\
  11201. leShardAction$ShardTransportHandler.messageReceived(TransportSingleSha\
  11202. rdAction.java:293)
  11203.  
  11204. at org.elasticsearch.action.support.single.shard.TransportSing\
  11205. leShardAction$ShardTransportHandler.messageReceived(TransportSingleSha\
  11206. rdAction.java:286)
  11207.  
  11208. at org.elasticsearch.transport.TransportRequestHandler.message\
  11209. Received(TransportRequestHandler.java:30)
  11210.  
  11211. at org.elasticsearch.transport.RequestHandlerRegistry.processM\
  11212. essageReceived(RequestHandlerRegistry.java:66)
  11213.  
  11214. at org.elasticsearch.transport.TcpTransport$RequestHandler.doR\
  11215. un(TcpTransport.java:1555)
  11216.  
  11217. at org.elasticsearch.common.util.concurrent.ThreadContext$Cont\
  11218. extPreservingAbstractRunnable.doRun(ThreadContext.java:635)
  11219.  
  11220. at org.elasticsearch.common.util.concurrent.AbstractRunnable.r\
  11221. un(AbstractRunnable.java:37)
  11222.  
  11223. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoo\
  11224. lExecutor.java:1149)
  11225.  
  11226. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPo\
  11227. olExecutor.java:624)
  11228.  
  11229. at java.lang.Thread.run(Thread.java:748)
  11230.  
  11231.  
  11232. Found 1 deadlock.
  11233.  
  11234.  
  11235. Heap
  11236.  
  11237. par new generation "
  11238. total 1380160K, used 486944K"
  11239. [0x0000000815200000, 0x0000000872b90000, 0x0000000872b90000)
  11240.  
  11241. eden"
  11242. space 1226816K, 32% used"
  11243. [0x0000000815200000, 0x000000082d58e6e8, 0x0000000860010000)
  11244.  
  11245. from"
  11246. space 153344K, 58% used"
  11247. [0x00000008695d0000, 0x000000086edc9ab0, 0x0000000872b90000)
  11248.  
  11249. to "
  11250. space 153344K, 0% used"
  11251. [0x0000000860010000, 0x0000000860010000, 0x00000008695d0000)
  11252.  
  11253. concurrent mark-sweep generation"
  11254. total 8952256K, used 2249292K"
  11255. [0x0000000872b90000, 0x0000000a95200000, 0x0000000a95200000)
  11256.  
  11257. Metaspace used 69014K, capacity 74912K, committed 75892K, reser\
  11258. ved 1116160K
  11259.  
  11260. class space used 8179K, capacity 9757K, committed 9996K, reserved\
  11261. 1048576K
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!