0> 2015-07-23 16:09:43.755199 7fa47ef00700 -1 os/FileStore.cc: In function 'virtual int FileStore::read(coll_t, const ghobject_t&, uint64_t, size_t, ceph::bufferlist&, uint32_t, bool)' thread 7fa47ef00700 time 2015-07-23 16:09:43.560568
os/FileStore.cc: 2850: FAILED assert(allow_eio || !m_filestore_fail_eio || got != -5)

 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3)
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0xbc2b8b]
 2: (FileStore::read(coll_t, ghobject_t const&, unsigned long, unsigned long, ceph::buffer::list&, unsigned int, bool)+0xc98) [0x916d58]
 3: (ReplicatedBackend::build_push_op(ObjectRecoveryInfo const&, ObjectRecoveryProgress const&, ObjectRecoveryProgress*, PushOp*, object_stat_sum_t*)+0x2c2) [0xa0da52]
 4: (ReplicatedBackend::handle_pull(pg_shard_t, PullOp&, PushOp*)+0xe4) [0xa10224]
 5: (ReplicatedBackend::do_pull(std::tr1::shared_ptr<OpRequest>)+0xd6) [0xa10506]
 6: (ReplicatedBackend::handle_message(std::tr1::shared_ptr<OpRequest>)+0x3ce) [0xa1763e]
 7: (ReplicatedPG::do_request(std::tr1::shared_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x167) [0x840f57]
 8: (OSD::dequeue_op(boost::intrusive_ptr<PG>, std::tr1::shared_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3d5) [0x6a0f85]
 9: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x331) [0x6a14d1]
 10: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x875) [0xbb2c85]
 11: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xbb4da0]
 12: (()+0x8182) [0x7fa4a3bfe182]
 13: (clone()+0x6d) [0x7fa4a216947d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
   0/ 5 none
   0/ 0 lockdep
   0/ 0 context
   0/ 0 crush
   1/ 5 mds
   1/ 5 mds_balancer
   1/ 5 mds_locker
   1/ 5 mds_log
   1/ 5 mds_log_expire
   1/ 5 mds_migrator
   0/ 0 buffer
   0/ 0 timer
   0/ 0 filer
   0/ 1 striper
   0/ 0 objecter
   0/ 0 rados
   0/ 0 rbd
   0/ 5 rbd_replay
   0/ 0 journaler
   0/ 5 objectcacher
   0/ 0 client
   0/ 0 osd
   0/ 0 optracker
   0/ 0 objclass
   0/ 0 filestore
   1/ 3 keyvaluestore
   0/ 0 journal
   0/ 0 ms
   0/ 0 mon
   0/ 0 monc
   0/ 0 paxos
   0/ 0 tp
   0/ 0 auth
   1/ 5 crypto
   0/ 0 finisher
   0/ 0 heartbeatmap
   0/ 0 perfcounter
   0/ 0 rgw
   1/10 civetweb
   1/ 5 javaclient
   0/ 0 asok
   0/ 0 throttle
   0/ 0 refs
   1/ 5 xio
  -2/-2 (syslog threshold)
  -1/-1 (stderr threshold)
  max_recent     10000
  max_new         1000
  log_file /var/log/ceph/ceph-osd.5.log
--- end dump of recent events ---
2015-07-23 16:09:43.804462 7fa47ef00700 -1 *** Caught signal (Aborted) **
 in thread 7fa47ef00700

 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3)
 1: /usr/bin/ceph-osd() [0xacb3ba]
 2: (()+0x10340) [0x7fa4a3c06340]
 3: (gsignal()+0x39) [0x7fa4a20a5cc9]
 4: (abort()+0x148) [0x7fa4a20a90d8]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x155) [0x7fa4a29b0535]
 6: (()+0x5e6d6) [0x7fa4a29ae6d6]
 7: (()+0x5e703) [0x7fa4a29ae703]
 8: (()+0x5e922) [0x7fa4a29ae922]
 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x278) [0xbc2d78]
 10: (FileStore::read(coll_t, ghobject_t const&, unsigned long, unsigned long, ceph::buffer::list&, unsigned int, bool)+0xc98) [0x916d58]
 11: (ReplicatedBackend::build_push_op(ObjectRecoveryInfo const&, ObjectRecoveryProgress const&, ObjectRecoveryProgress*, PushOp*, object_stat_sum_t*)+0x2c2) [0xa0da52]
 12: (ReplicatedBackend::handle_pull(pg_shard_t, PullOp&, PushOp*)+0xe4) [0xa10224]
 13: (ReplicatedBackend::do_pull(std::tr1::shared_ptr<OpRequest>)+0xd6) [0xa10506]
 14: (ReplicatedBackend::handle_message(std::tr1::shared_ptr<OpRequest>)+0x3ce) [0xa1763e]
 15: (ReplicatedPG::do_request(std::tr1::shared_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x167) [0x840f57]
 16: (OSD::dequeue_op(boost::intrusive_ptr<PG>, std::tr1::shared_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3d5) [0x6a0f85]
 17: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x331) [0x6a14d1]
 18: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x875) [0xbb2c85]
 19: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xbb4da0]
 20: (()+0x8182) [0x7fa4a3bfe182]
 21: (clone()+0x6d) [0x7fa4a216947d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- begin dump of recent events ---
     0> 2015-07-23 16:09:43.804462 7fa47ef00700 -1 *** Caught signal (Aborted) **
 in thread 7fa47ef00700

 ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3)
 1: /usr/bin/ceph-osd() [0xacb3ba]
 2: (()+0x10340) [0x7fa4a3c06340]
 3: (gsignal()+0x39) [0x7fa4a20a5cc9]
 4: (abort()+0x148) [0x7fa4a20a90d8]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x155) [0x7fa4a29b0535]
 6: (()+0x5e6d6) [0x7fa4a29ae6d6]
 7: (()+0x5e703) [0x7fa4a29ae703]
 8: (()+0x5e922) [0x7fa4a29ae922]
 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x278) [0xbc2d78]
 10: (FileStore::read(coll_t, ghobject_t const&, unsigned long, unsigned long, ceph::buffer::list&, unsigned int, bool)+0xc98) [0x916d58]
 11: (ReplicatedBackend::build_push_op(ObjectRecoveryInfo const&, ObjectRecoveryProgress const&, ObjectRecoveryProgress*, PushOp*, object_stat_sum_t*)+0x2c2) [0xa0da52]
 12: (ReplicatedBackend::handle_pull(pg_shard_t, PullOp&, PushOp*)+0xe4) [0xa10224]
 13: (ReplicatedBackend::do_pull(std::tr1::shared_ptr<OpRequest>)+0xd6) [0xa10506]
 14: (ReplicatedBackend::handle_message(std::tr1::shared_ptr<OpRequest>)+0x3ce) [0xa1763e]
 15: (ReplicatedPG::do_request(std::tr1::shared_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x167) [0x840f57]
 16: (OSD::dequeue_op(boost::intrusive_ptr<PG>, std::tr1::shared_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3d5) [0x6a0f85]
 17: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x331) [0x6a14d1]
 18: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x875) [0xbb2c85]
 19: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xbb4da0]
 20: (()+0x8182) [0x7fa4a3bfe182]
 21: (clone()+0x6d) [0x7fa4a216947d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
   0/ 5 none
   0/ 0 lockdep
   0/ 0 context
   0/ 0 crush
   1/ 5 mds
   1/ 5 mds_balancer
   1/ 5 mds_locker
   1/ 5 mds_log
   1/ 5 mds_log_expire
   1/ 5 mds_migrator
   0/ 0 buffer
   0/ 0 timer
   0/ 0 filer
   0/ 1 striper
   0/ 0 objecter
   0/ 0 rados
   0/ 0 rbd
   0/ 5 rbd_replay
   0/ 0 journaler
   0/ 5 objectcacher
   0/ 0 client
   0/ 0 osd
   0/ 0 optracker
   0/ 0 objclass
   0/ 0 filestore
   1/ 3 keyvaluestore
   0/ 0 journal
   0/ 0 ms
   0/ 0 mon
   0/ 0 monc
   0/ 0 paxos
   0/ 0 tp
   0/ 0 auth
   1/ 5 crypto
   0/ 0 finisher
   0/ 0 heartbeatmap
   0/ 0 perfcounter
   0/ 0 rgw
   1/10 civetweb
   1/ 5 javaclient
   0/ 0 asok
   0/ 0 throttle
   0/ 0 refs
   1/ 5 xio
  -2/-2 (syslog threshold)
  -1/-1 (stderr threshold)
  max_recent     10000
  max_new         1000
  log_file /var/log/ceph/ceph-osd.5.log
--- end dump of recent events ---