API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Jun 25th, 2017
567
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.65 KB | None | 0 0
raw download clone embed print report
  1. ./ceph.log-20170625.gz:78951:2017-06-24 23:11:08.177272 osd.179 10.124.103.74:6838/3737712 701 : cluster [ERR] 1.6df shard 29: soid 1:fb6b228b:::10007c8f674.00000000:head size 491400 != size 2056192 from auth oi 1:fb6b228b:::10007c8f674.00000000:head(193168'287263 mds.0.6882:335090848 dirty|omap_digest s 2056192 uv 287263 od ffffffff)
  2. ./ceph.log-20170625.gz:78952:2017-06-24 23:11:08.177282 osd.179 10.124.103.74:6838/3737712 702 : cluster [ERR] 1.6df shard 160: soid 1:fb6b228b:::10007c8f674.00000000:head size 491400 != size 2056192 from auth oi 1:fb6b228b:::10007c8f674.00000000:head(193168'287263 mds.0.6882:335090848 dirty|omap_digest s 2056192 uv 287263 od ffffffff)
  3. ./ceph.log-20170625.gz:78953:2017-06-24 23:11:08.177437 osd.179 10.124.103.74:6838/3737712 703 : cluster [ERR] 1.6df shard 179: soid 1:fb6b228b:::10007c8f674.00000000:head size 491400 != size 2056192 from auth oi 1:fb6b228b:::10007c8f674.00000000:head(193168'287263 mds.0.6882:335090848 dirty|omap_digest s 2056192 uv 287263 od ffffffff)
  4. ./ceph.log-20170625.gz:78954:2017-06-24 23:11:08.177441 osd.179 10.124.103.74:6838/3737712 704 : cluster [ERR] 1.6df soid 1:fb6b228b:::10007c8f674.00000000:head: failed to pick suitable auth object
  5.  
  6.  
  7.  
  8. pg = 1.6df
  9. osd = [171,160,29]
  10. object = 10007c8f674.00000000
  11.  
  12.  
  13.  
  14. find /var/lib/ceph/osd/ceph-171/current/1.6df_head/ -name '10007c8f674.00000000*' -ls
  15. [root@osd4 ceph-admin]# find /var/lib/ceph/osd/ceph-171/current/1.6df_head/ -name '10007c8f674.00000000*' -ls
  16. 2147899319 480 -rw-r--r-- 1 ceph ceph 2056192 Jun 25 09:19 /var/lib/ceph/osd/ceph-171/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  17.  
  18. find /var/lib/ceph/osd/ceph-160/current/1.6df_head/ -name '10007c8f674.00000000*' -ls
  19. [root@osd6 ceph]# find /var/lib/ceph/osd/ceph-160/current/1.6df_head/ -name '10007c8f674.00000000*' -ls
  20. 1074342919 480 -rw-r--r-- 1 ceph ceph 491400 Jun 22 15:23 /var/lib/ceph/osd/ceph-160/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  21.  
  22. find /var/lib/ceph/osd/ceph-29/current/1.6df_head/ -name '10007c8f674.00000000*' -ls
  23. [root@osd2 ceph]# find /var/lib/ceph/osd/ceph-29/current/1.6df_head/ -name '10007c8f674.00000000*' -ls
  24. 2684549007 480 -rw-r--r-- 1 ceph ceph 491400 Jun 22 15:23 /var/lib/ceph/osd/ceph-29/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  25.  
  26.  
  27. sha256sum /var/lib/ceph/osd/ceph-171/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  28. [root@osd4 ceph-admin]# sha256sum /var/lib/ceph/osd/ceph-171/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  29. 9a25c08c1dc4c37dedb39740c211121e0bb629458b88a8189268856017b521b3 /var/lib/ceph/osd/ceph-171/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  30.  
  31. sha256sum /var/lib/ceph/osd/ceph-160/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  32. [root@osd6 ceph]# sha256sum /var/lib/ceph/osd/ceph-160/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  33. 5a1525d825c8ac5cdd317bc5f95a3bd9eea2c2a30df0c57200bc2baa0f6f71ac /var/lib/ceph/osd/ceph-160/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  34.  
  35. sha256sum /var/lib/ceph/osd/ceph-29/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  36. [root@osd2 ceph]# sha256sum /var/lib/ceph/osd/ceph-29/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  37. 5a1525d825c8ac5cdd317bc5f95a3bd9eea2c2a30df0c57200bc2baa0f6f71ac /var/lib/ceph/osd/ceph-29/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1
  38.  
  39. #At this point we know that we have 1 good replica remaining. Going to backup the file on each osd node.
  40.  
  41. cp /var/lib/ceph/osd/ceph-171/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1 ~/
  42. cp /var/lib/ceph/osd/ceph-160/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1 ~/
  43. cp /var/lib/ceph/osd/ceph-29/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1 ~/
  44.  
  45. [root@osd2 ceph]# ceph osd set noout
  46. set noout
  47.  
  48. [root@osd2 ceph]# systemctl stop ceph-osd@29.service
  49. [root@osd4 ceph-admin]# systemctl stop ceph-osd@171.service
  50. [root@osd6 ceph]# systemctl stop ceph-osd@160.service
  51.  
  52. ceph-osd -i 29 --flush-journal
  53. SG_IO: questionable sense data, results may be incorrect
  54. SG_IO: questionable sense data, results may be incorrect
  55. 2017-06-25 21:24:46.078299 7f52fb4c7800 -1 flushed journal /var/lib/ceph/osd/ceph-29/journal for object store /var/lib/ceph/osd/ceph-29
  56.  
  57.  
  58. ceph-osd -i 171 --flush-journal
  59. SG_IO: questionable sense data, results may be incorrect
  60. SG_IO: questionable sense data, results may be incorrect
  61. 2017-06-25 21:25:35.887907 7fed391b0800 -1 flushed journal /var/lib/ceph/osd/ceph-171/journal for object store /var/lib/ceph/osd/ceph-171
  62.  
  63.  
  64. ceph-osd -i 160 --flush-journal
  65. [root@osd6 ceph]# ceph-osd -i 160 --flush-journal
  66. SG_IO: questionable sense data, results may be incorrect
  67. SG_IO: questionable sense data, results may be incorrect
  68. *** Caught signal (Segmentation fault) **
  69. in thread 7f8bdccf7700 thread_name:ceph-osd
  70. ceph version 10.2.7 (50e863e0f4bc8f4b9e31156de690d765af245185)
  71. 1: (()+0x91d8ea) [0x7f8be3b448ea]
  72. 2: (()+0xf100) [0x7f8be1960100]
  73. 2017-06-25 21:26:04.686577 7f8bdccf7700 -1 *** Caught signal (Segmentation fault) **
  74. in thread 7f8bdccf7700 thread_name:ceph-osd
  75.  
  76. ceph version 10.2.7 (50e863e0f4bc8f4b9e31156de690d765af245185)
  77. 1: (()+0x91d8ea) [0x7f8be3b448ea]
  78. 2: (()+0xf100) [0x7f8be1960100]
  79. NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  80.  
  81. 0> 2017-06-25 21:26:04.686577 7f8bdccf7700 -1 *** Caught signal (Segmentation fault) **
  82. in thread 7f8bdccf7700 thread_name:ceph-osd
  83.  
  84. ceph version 10.2.7 (50e863e0f4bc8f4b9e31156de690d765af245185)
  85. 1: (()+0x91d8ea) [0x7f8be3b448ea]
  86. 2: (()+0xf100) [0x7f8be1960100]
  87. NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  88.  
  89. Segmentation fault
  90.  
  91.  
  92.  
  93. cp /var/lib/ceph/osd/ceph-171/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1 ~/10007c8f674.00000000__head_D144D6DF__1--post_journal_flush
  94. cp /var/lib/ceph/osd/ceph-160/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1 ~/10007c8f674.00000000__head_D144D6DF__1--post_journal_flush
  95. cp /var/lib/ceph/osd/ceph-29/current/1.6df_head/DIR_F/DIR_D/DIR_6/DIR_D/DIR_4/10007c8f674.00000000__head_D144D6DF__1 ~/10007c8f674.00000000__head_D144D6DF__1--post_journal_flush
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!