API tools faq
paste
jdwx

OSD console output

jdwx
Aug 1st, 2018
298
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 14.96 KB | None | 0 0
raw download clone embed print report
  1. $ sudo /usr/bin/ceph-osd -f --cluster ceph --id 14 --setuser ceph --setgroup ceph
  2. starting osd.14 at - osd_data /var/lib/ceph/osd/ceph-14 /var/lib/ceph/osd/ceph-14/journal
  3. 2018-08-01 12:17:09.540647 7fb530770e00 -1 journal do_read_entry(645697536): bad header magic
  4. 2018-08-01 12:17:20.451805 7fb530770e00 -1 osd.14 22817 log_to_monitors {default=true}
  5. 2018-08-01 12:17:20.584350 7fb50eded700 -1 log_channel(cluster) log [ERR] : 2.621 past_interal bound [19300,21449) end does not match required [21374,21447) end
  6. /build/ceph-12.2.7/src/osd/PG.cc: In function 'void PG::check_past_interval_bounds() const' thread 7fb50eded700 time 2018-08-01 12:17:20.584367
  7. /build/ceph-12.2.7/src/osd/PG.cc: 847: FAILED assert(0 == "past_interval end mismatch")
  8. 2018-08-01 12:17:20.584356 7fb50eded700 -1 osd.14 pg_epoch: 22806 pg[2.621( v 21191'1231760 (19460'1230260,21191'1231760] lb MIN (bitwise) local-lis/les=21447/21449 n=0 ec=1/1 lis/c 21447/21374 les/c/f 21449/21374/0 21443/21447/21393) [17,14]/[17] r=-1 lpr=22805 pi=[19300,21449)/172 crt=21191'1231760 lcod 0'0 unknown NOTIFY] 2.621 past_interal bound [19300,21449) end does not match required [21374,21447) end
  9. ceph version 12.2.7 (3ec878d1e53e1aeb47a9f619c49d9e7c0aa384d5) luminous (stable)
  10. 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x102) [0x55bd0ff69232]
  11. 2: (PG::check_past_interval_bounds() const+0x794) [0x55bd0f9f0c54]
  12. 3: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x4bb) [0x55bd0fa302ab]
  13. 4: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x188) [0x55bd0fa7c2d8]
  14. 5: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x69) [0x55bd0fa531b9]
  15. 6: (PG::handle_advance_map(std::shared_ptr<OSDMap const>, std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, int, std::vector<int, std::allocator<int> >&, int, PG::RecoveryCtx*)+0x4a7) [0x55bd0fa134e7]
  16. 7: (OSD::advance_pg(unsigned int, PG*, ThreadPool::TPHandle&, PG::RecoveryCtx*, std::set<boost::intrusive_ptr<PG>, std::less<boost::intrusive_ptr<PG> >, std::allocator<boost::intrusive_ptr<PG> > >*)+0x2e7) [0x55bd0f95a4e7]
  17. 8: (OSD::process_peering_events(std::__cxx11::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x1e4) [0x55bd0f95b334]
  18. 9: (ThreadPool::BatchWorkQueue<PG>::_void_process(void*, ThreadPool::TPHandle&)+0x2c) [0x55bd0f9cb60c]
  19. 10: (ThreadPool::worker(ThreadPool::WorkThread*)+0xeb8) [0x55bd0ff703d8]
  20. 11: (ThreadPool::WorkThread::entry()+0x10) [0x55bd0ff71570]
  21. 12: (()+0x7494) [0x7fb52df7e494]
  22. 13: (clone()+0x3f) [0x7fb52d005acf]
  23. NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  24. 2018-08-01 12:17:20.587975 7fb50eded700 -1 /build/ceph-12.2.7/src/osd/PG.cc: In function 'void PG::check_past_interval_bounds() const' thread 7fb50eded700 time 2018-08-01 12:17:20.584367
  25. /build/ceph-12.2.7/src/osd/PG.cc: 847: FAILED assert(0 == "past_interval end mismatch")
  26.  
  27. ceph version 12.2.7 (3ec878d1e53e1aeb47a9f619c49d9e7c0aa384d5) luminous (stable)
  28. 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x102) [0x55bd0ff69232]
  29. 2: (PG::check_past_interval_bounds() const+0x794) [0x55bd0f9f0c54]
  30. 3: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x4bb) [0x55bd0fa302ab]
  31. 4: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x188) [0x55bd0fa7c2d8]
  32. 5: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x69) [0x55bd0fa531b9]
  33. 6: (PG::handle_advance_map(std::shared_ptr<OSDMap const>, std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, int, std::vector<int, std::allocator<int> >&, int, PG::RecoveryCtx*)+0x4a7) [0x55bd0fa134e7]
  34. 7: (OSD::advance_pg(unsigned int, PG*, ThreadPool::TPHandle&, PG::RecoveryCtx*, std::set<boost::intrusive_ptr<PG>, std::less<boost::intrusive_ptr<PG> >, std::allocator<boost::intrusive_ptr<PG> > >*)+0x2e7) [0x55bd0f95a4e7]
  35. 8: (OSD::process_peering_events(std::__cxx11::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x1e4) [0x55bd0f95b334]
  36. 9: (ThreadPool::BatchWorkQueue<PG>::_void_process(void*, ThreadPool::TPHandle&)+0x2c) [0x55bd0f9cb60c]
  37. 10: (ThreadPool::worker(ThreadPool::WorkThread*)+0xeb8) [0x55bd0ff703d8]
  38. 11: (ThreadPool::WorkThread::entry()+0x10) [0x55bd0ff71570]
  39. 12: (()+0x7494) [0x7fb52df7e494]
  40. 13: (clone()+0x3f) [0x7fb52d005acf]
  41. NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  42.  
  43. -3974> 2018-08-01 12:17:09.540647 7fb530770e00 -1 journal do_read_entry(645697536): bad header magic
  44. -2230> 2018-08-01 12:17:20.451805 7fb530770e00 -1 osd.14 22817 log_to_monitors {default=true}
  45. -44> 2018-08-01 12:17:20.584350 7fb50eded700 -1 log_channel(cluster) log [ERR] : 2.621 past_interal bound [19300,21449) end does not match required [21374,21447) end
  46. -43> 2018-08-01 12:17:20.584356 7fb50eded700 -1 osd.14 pg_epoch: 22806 pg[2.621( v 21191'1231760 (19460'1230260,21191'1231760] lb MIN (bitwise) local-lis/les=21447/21449 n=0 ec=1/1 lis/c 21447/21374 les/c/f 21449/21374/0 21443/21447/21393) [17,14]/[17] r=-1 lpr=22805 pi=[19300,21449)/172 crt=21191'1231760 lcod 0'0 unknown NOTIFY] 2.621 past_interal bound [19300,21449) end does not match required [21374,21447) end
  47. 0> 2018-08-01 12:17:20.587975 7fb50eded700 -1 /build/ceph-12.2.7/src/osd/PG.cc: In function 'void PG::check_past_interval_bounds() const' thread 7fb50eded700 time 2018-08-01 12:17:20.584367
  48. /build/ceph-12.2.7/src/osd/PG.cc: 847: FAILED assert(0 == "past_interval end mismatch")
  49.  
  50. ceph version 12.2.7 (3ec878d1e53e1aeb47a9f619c49d9e7c0aa384d5) luminous (stable)
  51. 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x102) [0x55bd0ff69232]
  52. 2: (PG::check_past_interval_bounds() const+0x794) [0x55bd0f9f0c54]
  53. 3: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x4bb) [0x55bd0fa302ab]
  54. 4: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x188) [0x55bd0fa7c2d8]
  55. 5: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x69) [0x55bd0fa531b9]
  56. 6: (PG::handle_advance_map(std::shared_ptr<OSDMap const>, std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, int, std::vector<int, std::allocator<int> >&, int, PG::RecoveryCtx*)+0x4a7) [0x55bd0fa134e7]
  57. 7: (OSD::advance_pg(unsigned int, PG*, ThreadPool::TPHandle&, PG::RecoveryCtx*, std::set<boost::intrusive_ptr<PG>, std::less<boost::intrusive_ptr<PG> >, std::allocator<boost::intrusive_ptr<PG> > >*)+0x2e7) [0x55bd0f95a4e7]
  58. 8: (OSD::process_peering_events(std::__cxx11::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x1e4) [0x55bd0f95b334]
  59. 9: (ThreadPool::BatchWorkQueue<PG>::_void_process(void*, ThreadPool::TPHandle&)+0x2c) [0x55bd0f9cb60c]
  60. 10: (ThreadPool::worker(ThreadPool::WorkThread*)+0xeb8) [0x55bd0ff703d8]
  61. 11: (ThreadPool::WorkThread::entry()+0x10) [0x55bd0ff71570]
  62. 12: (()+0x7494) [0x7fb52df7e494]
  63. 13: (clone()+0x3f) [0x7fb52d005acf]
  64. NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  65.  
  66. *** Caught signal (Aborted) **
  67. in thread 7fb50eded700 thread_name:tp_peering
  68. ceph version 12.2.7 (3ec878d1e53e1aeb47a9f619c49d9e7c0aa384d5) luminous (stable)
  69. 1: (()+0xa394f4) [0x55bd0ff214f4]
  70. 2: (()+0x110c0) [0x7fb52df880c0]
  71. 3: (gsignal()+0xcf) [0x7fb52cf4ffff]
  72. 4: (abort()+0x16a) [0x7fb52cf5142a]
  73. 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x28e) [0x55bd0ff693be]
  74. 6: (PG::check_past_interval_bounds() const+0x794) [0x55bd0f9f0c54]
  75. 7: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x4bb) [0x55bd0fa302ab]
  76. 8: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x188) [0x55bd0fa7c2d8]
  77. 9: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x69) [0x55bd0fa531b9]
  78. 10: (PG::handle_advance_map(std::shared_ptr<OSDMap const>, std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, int, std::vector<int, std::allocator<int> >&, int, PG::RecoveryCtx*)+0x4a7) [0x55bd0fa134e7]
  79. 11: (OSD::advance_pg(unsigned int, PG*, ThreadPool::TPHandle&, PG::RecoveryCtx*, std::set<boost::intrusive_ptr<PG>, std::less<boost::intrusive_ptr<PG> >, std::allocator<boost::intrusive_ptr<PG> > >*)+0x2e7) [0x55bd0f95a4e7]
  80. 12: (OSD::process_peering_events(std::__cxx11::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x1e4) [0x55bd0f95b334]
  81. 13: (ThreadPool::BatchWorkQueue<PG>::_void_process(void*, ThreadPool::TPHandle&)+0x2c) [0x55bd0f9cb60c]
  82. 14: (ThreadPool::worker(ThreadPool::WorkThread*)+0xeb8) [0x55bd0ff703d8]
  83. 15: (ThreadPool::WorkThread::entry()+0x10) [0x55bd0ff71570]
  84. 16: (()+0x7494) [0x7fb52df7e494]
  85. 17: (clone()+0x3f) [0x7fb52d005acf]
  86. 2018-08-01 12:17:20.612820 7fb50eded700 -1 *** Caught signal (Aborted) **
  87. in thread 7fb50eded700 thread_name:tp_peering
  88.  
  89. ceph version 12.2.7 (3ec878d1e53e1aeb47a9f619c49d9e7c0aa384d5) luminous (stable)
  90. 1: (()+0xa394f4) [0x55bd0ff214f4]
  91. 2: (()+0x110c0) [0x7fb52df880c0]
  92. 3: (gsignal()+0xcf) [0x7fb52cf4ffff]
  93. 4: (abort()+0x16a) [0x7fb52cf5142a]
  94. 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x28e) [0x55bd0ff693be]
  95. 6: (PG::check_past_interval_bounds() const+0x794) [0x55bd0f9f0c54]
  96. 7: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x4bb) [0x55bd0fa302ab]
  97. 8: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x188) [0x55bd0fa7c2d8]
  98. 9: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x69) [0x55bd0fa531b9]
  99. 10: (PG::handle_advance_map(std::shared_ptr<OSDMap const>, std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, int, std::vector<int, std::allocator<int> >&, int, PG::RecoveryCtx*)+0x4a7) [0x55bd0fa134e7]
  100. 11: (OSD::advance_pg(unsigned int, PG*, ThreadPool::TPHandle&, PG::RecoveryCtx*, std::set<boost::intrusive_ptr<PG>, std::less<boost::intrusive_ptr<PG> >, std::allocator<boost::intrusive_ptr<PG> > >*)+0x2e7) [0x55bd0f95a4e7]
  101. 12: (OSD::process_peering_events(std::__cxx11::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x1e4) [0x55bd0f95b334]
  102. 13: (ThreadPool::BatchWorkQueue<PG>::_void_process(void*, ThreadPool::TPHandle&)+0x2c) [0x55bd0f9cb60c]
  103. 14: (ThreadPool::worker(ThreadPool::WorkThread*)+0xeb8) [0x55bd0ff703d8]
  104. 15: (ThreadPool::WorkThread::entry()+0x10) [0x55bd0ff71570]
  105. 16: (()+0x7494) [0x7fb52df7e494]
  106. 17: (clone()+0x3f) [0x7fb52d005acf]
  107. NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  108.  
  109. 0> 2018-08-01 12:17:20.612820 7fb50eded700 -1 *** Caught signal (Aborted) **
  110. in thread 7fb50eded700 thread_name:tp_peering
  111.  
  112. ceph version 12.2.7 (3ec878d1e53e1aeb47a9f619c49d9e7c0aa384d5) luminous (stable)
  113. 1: (()+0xa394f4) [0x55bd0ff214f4]
  114. 2: (()+0x110c0) [0x7fb52df880c0]
  115. 3: (gsignal()+0xcf) [0x7fb52cf4ffff]
  116. 4: (abort()+0x16a) [0x7fb52cf5142a]
  117. 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x28e) [0x55bd0ff693be]
  118. 6: (PG::check_past_interval_bounds() const+0x794) [0x55bd0f9f0c54]
  119. 7: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x4bb) [0x55bd0fa302ab]
  120. 8: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x188) [0x55bd0fa7c2d8]
  121. 9: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x69) [0x55bd0fa531b9]
  122. 10: (PG::handle_advance_map(std::shared_ptr<OSDMap const>, std::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, int, std::vector<int, std::allocator<int> >&, int, PG::RecoveryCtx*)+0x4a7) [0x55bd0fa134e7]
  123. 11: (OSD::advance_pg(unsigned int, PG*, ThreadPool::TPHandle&, PG::RecoveryCtx*, std::set<boost::intrusive_ptr<PG>, std::less<boost::intrusive_ptr<PG> >, std::allocator<boost::intrusive_ptr<PG> > >*)+0x2e7) [0x55bd0f95a4e7]
  124. 12: (OSD::process_peering_events(std::__cxx11::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x1e4) [0x55bd0f95b334]
  125. 13: (ThreadPool::BatchWorkQueue<PG>::_void_process(void*, ThreadPool::TPHandle&)+0x2c) [0x55bd0f9cb60c]
  126. 14: (ThreadPool::worker(ThreadPool::WorkThread*)+0xeb8) [0x55bd0ff703d8]
  127. 15: (ThreadPool::WorkThread::entry()+0x10) [0x55bd0ff71570]
  128. 16: (()+0x7494) [0x7fb52df7e494]
  129. 17: (clone()+0x3f) [0x7fb52d005acf]
  130. NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
  131.  
  132. Aborted
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!