-50> 2012-11-29 08:01:30.667479 7fc427dfd700  2 mds.1.4 i am not alone, moving to state resolve
   -49> 2012-11-29 08:01:30.667519 7fc427dfd700  3 mds.1.4 request_state up:resolve
   -48> 2012-11-29 08:01:30.667580 7fc427dfd700  1 -- 192.168.2.81:6800/15765 --> 192.168.2.80:6789/0 -- mdsbeacon(4183/baker up:resolve seq 9 v318) v2 -- ?+0 0x7fc40c8dbae0 con 0x27095f0
   -47> 2012-11-29 08:01:30.667699 7fc44a7fc700  5 throttle(objecter_bytes 0x2705f08) put 0 (0xad01c8 -> 0)
   -46> 2012-11-29 08:01:30.667721 7fc44a7fc700  5 throttle(objecter_ops 0x2705f88) put 1 (0xad01c8 -> 5)
   -45> 2012-11-29 08:01:30.667774 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 111 (0xad01c8 -> 222)
   -44> 2012-11-29 08:01:30.667788 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== osd.2 192.168.2.81:6801/4827 21 ==== osd_op_reply(69 201.00000040 [delete] ondisk = -2 (No such file or directory)) v4 ===
= 111+0+0 (463046018 0 0) 0x7fc410001b90 con 0x7fc430015f80
   -43> 2012-11-29 08:01:30.667834 7fc44a7fc700  5 throttle(objecter_bytes 0x2705f08) put 0 (0xad01c8 -> 0)
   -42> 2012-11-29 08:01:30.667840 7fc44a7fc700  5 throttle(objecter_ops 0x2705f88) put 1 (0xad01c8 -> 4)
   -41> 2012-11-29 08:01:30.667858 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 111 (0xad01c8 -> 111)
   -40> 2012-11-29 08:01:30.667869 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== osd.2 192.168.2.81:6801/4827 22 ==== osd_op_reply(71 201.00000042 [delete] ondisk = -2 (No such file or directory)) v4 ===
= 111+0+0 (2413372171 0 0) 0x7fc4100023f0 con 0x7fc430015f80
   -39> 2012-11-29 08:01:30.667902 7fc44a7fc700  5 throttle(objecter_bytes 0x2705f08) put 0 (0xad01c8 -> 0)
   -38> 2012-11-29 08:01:30.667908 7fc44a7fc700  5 throttle(objecter_ops 0x2705f88) put 1 (0xad01c8 -> 3)
   -37> 2012-11-29 08:01:30.667922 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 111 (0xad01c8 -> 0)
   -36> 2012-11-29 08:01:30.672394 7fc4484f5700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) get 111 (0 -> 111)
   -35> 2012-11-29 08:01:30.672653 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== osd.1 192.168.2.80:6804/20043 15 ==== osd_op_reply(72 201.00000043 [delete] ondisk = -2 (No such file or directory)) v4 ==
== 111+0+0 (3801541718 0 0) 0x7fc40d4c0ab0 con 0x7fc430014350
   -34> 2012-11-29 08:01:30.672791 7fc44a7fc700  5 throttle(objecter_bytes 0x2705f08) put 0 (0xad01c8 -> 0)
   -33> 2012-11-29 08:01:30.672803 7fc44a7fc700  5 throttle(objecter_ops 0x2705f88) put 1 (0xad01c8 -> 2)
   -32> 2012-11-29 08:01:30.672831 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 111 (0xad01c8 -> 0)
   -31> 2012-11-29 08:01:30.672930 7fc4484f5700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) get 111 (0 -> 111)
   -30> 2012-11-29 08:01:30.673124 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== osd.1 192.168.2.80:6804/20043 16 ==== osd_op_reply(73 201.00000044 [delete] ondisk = -2 (No such file or directory)) v4 ==
== 111+0+0 (3706834655 0 0) 0x7fc40c8bd6e0 con 0x7fc430014350
   -29> 2012-11-29 08:01:30.673181 7fc44a7fc700  5 throttle(objecter_bytes 0x2705f08) put 0 (0xad01c8 -> 0)
   -28> 2012-11-29 08:01:30.673191 7fc44a7fc700  5 throttle(objecter_ops 0x2705f88) put 1 (0xad01c8 -> 1)
   -27> 2012-11-29 08:01:30.673208 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 111 (0xad01c8 -> 0)
   -26> 2012-11-29 08:01:30.787132 7fc4484f5700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) get 111 (0 -> 111)
   -25> 2012-11-29 08:01:30.787402 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== osd.1 192.168.2.80:6804/20043 17 ==== osd_op_reply(68 201.0000003f [truncate 3686912] ondisk = 0) v4 ==== 111+0+0 (1891080
899 0 0) 0x7fc40d682050 con 0x7fc430014350
   -24> 2012-11-29 08:01:30.787498 7fc44a7fc700  5 throttle(objecter_bytes 0x2705f08) put 0 (0xad01c8 -> 0)
   -23> 2012-11-29 08:01:30.787508 7fc44a7fc700  5 throttle(objecter_ops 0x2705f88) put 1 (0xad01c8 -> 0)
   -22> 2012-11-29 08:01:30.787538 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 111 (0xad01c8 -> 0)
   -21> 2012-11-29 08:01:30.901231 7fc4501f4700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) get 765 (0 -> 765)
   -20> 2012-11-29 08:01:30.901383 7fc4501f4700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) get 56 (765 -> 821)
   -19> 2012-11-29 08:01:30.901469 7fc4502f5700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) get 765 (821 -> 1586)
   -18> 2012-11-29 08:01:30.901526 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== mds.0 192.168.2.80:6800/19829 3 ==== mdsmap(e 319) v1 ==== 765+0+0 (3805314058 0 0) 0x7fc4201133e0 con 0x7fc440002a10
   -17> 2012-11-29 08:01:30.901643 7fc44a7fc700  5 mds.1.4 handle_mds_map epoch 319 from mds.0
   -16> 2012-11-29 08:01:30.901804 7fc44a7fc700  1 mds.1.4 handle_mds_map i am now mds.1.4
   -15> 2012-11-29 08:01:30.901818 7fc44a7fc700  1 mds.1.4 handle_mds_map state change up:replay --> up:resolve
   -14> 2012-11-29 08:01:30.901826 7fc44a7fc700  1 mds.1.4 resolve_start
   -13> 2012-11-29 08:01:30.901831 7fc44a7fc700  1 mds.1.4 reopen_log
   -12> 2012-11-29 08:01:30.901864 7fc44a7fc700  1 mds.1.4  recovery set is 0
   -11> 2012-11-29 08:01:30.901898 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 --> mds.0 192.168.2.80:6800/19829 -- mds_resolve(2+0 subtrees +1 slave requests) v1 -- ?+0 0x7fc43003b770
   -10> 2012-11-29 08:01:30.902020 7fc44a7fc700  5 mds.1.bal rebalance done
    -9> 2012-11-29 08:01:30.904696 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 765 (0xad01c8 -> 821)
    -8> 2012-11-29 08:01:30.904738 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== mon.0 192.168.2.80:6789/0 23 ==== mdsmap(e 319) v1 ==== 765+0+0 (3805314058 0 0) 0x7fc434001e00 con 0x27095f0
    -7> 2012-11-29 08:01:30.904782 7fc44a7fc700  5 mds.1.4 handle_mds_map epoch 319 from mon.0
    -6> 2012-11-29 08:01:30.904791 7fc44a7fc700  5 mds.1.4  old map epoch 319 <= 319, discarding
    -5> 2012-11-29 08:01:30.904807 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 765 (0xad01c8 -> 56)
    -4> 2012-11-29 08:01:30.904820 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== mds.0 192.168.2.80:6800/19829 4 ==== mds_resolve(2+0 subtrees +0 slave requests) v1 ==== 56+0+0 (2566832879 0 0) 0x7fc4201
139e0 con 0x7fc440002a10
    -3> 2012-11-29 08:01:30.904932 7fc44a7fc700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) put 56 (0xad01c8 -> 0)
    -2> 2012-11-29 08:01:30.906617 7fc4501f4700  5 throttle(msgr_dispatch_throttler-mds 0x26eade8) get 25 (0 -> 25)
    -1> 2012-11-29 08:01:30.906884 7fc44a7fc700  1 -- 192.168.2.81:6800/15765 <== mds.0 192.168.2.80:6800/19829 5 ==== resolve_ack v1 ==== 25+0+0 (2010715686 0 0) 0x7fc4201130d0 con 0x7fc440002a10
     0> 2012-11-29 08:01:30.911291 7fc44a7fc700 -1 mds/Server.cc: In function 'void Server::do_rename_rollback(ceph::bufferlist&, int, MDRequest*)' thread 7fc44a7fc700 time 2012-11-29 08:01:30.907016
mds/Server.cc: 6286: FAILED assert(srcdir)

 ceph version 0.54 (commit:60b84b095b1009a305d4d6a5b16f88571cbd3150)
 1: (Server::do_rename_rollback(ceph::buffer::list&, int, MDRequest*)+0xb81) [0x5066f1]
 2: (MDCache::handle_resolve_ack(MMDSResolveAck*)+0x116b) [0x5b31db]
 3: (MDCache::dispatch(Message*)+0x145) [0x5ba315]
 4: (MDS::handle_deferrable_message(Message*)+0x6a8) [0x4a7ca8]
 5: (MDS::_dispatch(Message*)+0x983) [0x4b99a3]
 6: (MDS::ms_dispatch(Message*)+0x21b) [0x4bacdb]
 7: (DispatchQueue::entry()+0x699) [0x7c4fa9]
 8: (DispatchQueue::DispatchThread::entry()+0xd) [0x7c293d]
 9: /lib64/libpthread.so.0() [0x3182407851]
 10: (clone()+0x6d) [0x31820e811d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
   0/ 5 none
   0/ 5 lockdep
   0/ 5 context
   1/ 5 crush
   1/ 5 mds
   1/ 5 mds_balancer
   1/ 5 mds_locker
   1/ 5 mds_log
   1/ 5 mds_log_expire
   1/ 5 mds_migrator
   0/ 0 buffer
   0/ 5 timer
   0/ 5 filer
   0/ 0 objecter
   0/ 5 rados
   0/ 5 rbd

   0/ 5 journaler
   0/ 5 objectcacher
   0/ 5 client
   0/ 5 osd
   0/ 5 optracker
   0/ 5 objclass
   1/ 5 filestore
   1/ 5 journal
   0/ 5 ms
   1/ 5 mon
   0/ 5 monc
   0/ 5 paxos
   0/ 5 tp
   1/ 5 auth
   1/ 5 finisher
   1/ 5 heartbeatmap
   1/ 5 perfcounter
   1/ 5 rgw
   1/ 5 hadoop
   1/ 5 asok
   1/ 5 throttle
  -2/-2 (syslog threshold)
  99/99 (stderr threshold)
  max_recent     10000
  max_new      1000000
  log_file
--- end dump of recent events ---
terminate called after throwing an instance of 'ceph::FailedAssertion'
*** Caught signal (Aborted) **
 in thread 7fc44a7fc700
 ceph version 0.54 (commit:60b84b095b1009a305d4d6a5b16f88571cbd3150)
 1: ceph-mds() [0x7f9d01]
 2: /lib64/libpthread.so.0() [0x318240f500]
 3: (gsignal()+0x35) [0x31820328a5]
 4: (abort()+0x175) [0x3182034085]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x12d) [0x3183cbea5d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- begin dump of recent events ---
     0> 2012-11-29 08:01:32.101456 7fc44a7fc700 -1 *** Caught signal (Aborted) **
 in thread 7fc44a7fc700

 ceph version 0.54 (commit:60b84b095b1009a305d4d6a5b16f88571cbd3150)
 1: ceph-mds() [0x7f9d01]
 2: /lib64/libpthread.so.0() [0x318240f500]
 3: (gsignal()+0x35) [0x31820328a5]
 4: (abort()+0x175) [0x3182034085]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x12d) [0x3183cbea5d]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
   0/ 5 none
   0/ 5 lockdep
   0/ 5 context
   1/ 5 crush
   1/ 5 mds
   1/ 5 mds_balancer
   1/ 5 mds_locker
   1/ 5 mds_log
   1/ 5 mds_log_expire
   1/ 5 mds_migrator
   0/ 0 buffer
   0/ 5 timer
   0/ 5 filer
   0/ 0 objecter
   0/ 5 rados
   0/ 5 rbd
   0/ 5 journaler
   0/ 5 objectcacher
   0/ 5 client
   0/ 5 osd
   0/ 5 optracker
   0/ 5 objclass
   1/ 5 filestore
   1/ 5 journal
   0/ 5 ms
   1/ 5 mon
   0/ 5 monc
   0/ 5 paxos
   0/ 5 tp
   1/ 5 auth
   1/ 5 finisher
   1/ 5 heartbeatmap
   1/ 5 perfcounter
   1/ 5 rgw
   1/ 5 hadoop
   1/ 5 asok
   1/ 5 throttle
  -2/-2 (syslog threshold)
  99/99 (stderr threshold)
  max_recent     10000
  max_new      1000000
  log_file
--- end dump of recent events ---
Aborted