ceph osd debug

-- OSD log

2013-02-13 19:39:23.228014 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:23.228062 7f76807d7700 20 osd.0 10 scrub_should_schedule loadavg 0.01 < max 0.5 = yes
2013-02-13 19:39:23.228106 7f76807d7700 20 osd.0 10 sched_scrub load_is_low=1
2013-02-13 19:39:23.229709 7f76807d7700 20 osd.0 10 sched_scrub done
2013-02-13 19:39:24.229785 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:24.229804 7f76807d7700 20 osd.0 10 scrub_random_backoff lost coin flip, randomly backing off
2013-02-13 19:39:24.229809 7f76807d7700  7 osd.0 10 do_mon_report
2013-02-13 19:39:24.229812 7f76807d7700 10 osd.0 10 send_alive up_thru currently 7 want 7
2013-02-13 19:39:24.229815 7f76807d7700 20 osd.0 10 send_pg_stats
2013-02-13 19:39:24.777340 7f76717b9700 20 osd.0 10 update_osd_stat osd_stat(195 GB used, 1561 GB avail, 1830 GB total, peers []/[])
2013-02-13 19:39:24.777358 7f76717b9700  5 osd.0 10 heartbeat: osd_stat(195 GB used, 1561 GB avail, 1830 GB total, peers []/[])
2013-02-13 19:39:25.229889 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:25.229907 7f76807d7700 20 osd.0 10 scrub_random_backoff lost coin flip, randomly backing off
2013-02-13 19:39:25.467879 7f766fdb4700 10 osd.0 10  new session 0x2c8cc60 con=0x25074a0 addr=10.x.y.z:0/1004293
2013-02-13 19:39:25.467916 7f766fdb4700 10 osd.0 10  session 0x2c8cc60 client.libvirt has caps osdcap[grant(object_prefix rbd^@children  class-read),grant(pool libvirt^@pool^@test rwx)] 'allow class-read object_prefix rbd_children, allow pool libvirt-pool-test rwx'
2013-02-13 19:39:25.468162 7f76787c7700 20 osd.0 10 _dispatch 0x3960240 osd_op(client.4254.0:1 rbd_directory [read 0~0] 3.30a98c1c) v4
2013-02-13 19:39:25.468218 7f76787c7700 15 osd.0 10 require_same_or_newer_map 10 (i am 10) 0x3960240
2013-02-13 19:39:25.468223 7f76787c7700 20 osd.0 10 _share_map_incoming client.4254 10.x.y.z:0/1004293 10
2013-02-13 19:39:25.468232 7f76787c7700 15 osd.0 10 enqueue_op 0x334cb40 prio 63 cost 0 latency 0.000160 osd_op(client.4254.0:1 rbd_directory [read 0~0] 3.30a98c1c) v4
2013-02-13 19:39:25.468266 7f76737bd700 10 osd.0 10 dequeue_op 0x334cb40 prio 63 cost 0 latency 0.000195 osd_op(client.4254.0:1 rbd_directory [read 0~0] 3.30a98c1c) v4 pg pg[3.1c( v 10'2 (0'0,10'2] local-les=8 n=1 ec=4 les/c 8/8 7/7/7) [0] r=0 lpr=7 mlcod 10'2 active+degraded]
2013-02-13 19:39:25.468312 7f76737bd700 20 osd.0 pg_epoch: 10 pg[3.1c( v 10'2 (0'0,10'2] local-les=8 n=1 ec=4 les/c 8/8 7/7/7) [0] r=0 lpr=7 mlcod 10'2 active+degraded] op_has_sufficient_caps pool=3 (libvirt-pool-test) owner=0 need_read_cap=1 need_write_cap=0 need_class_read_cap=0 need_class_write_cap=0 -> NO
2013-02-13 19:39:25.468358 7f76737bd700 10 osd.0 10 dequeue_op 0x334cb40 finish
2013-02-13 19:39:25.469124 7f76787c7700  1 osd.0 10 OSD::ms_handle_reset()
2013-02-13 19:39:26.229985 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:26.230004 7f76807d7700 20 osd.0 10 scrub_random_backoff lost coin flip, randomly backing off
2013-02-13 19:39:26.477486 7f76717b9700 20 osd.0 10 update_osd_stat osd_stat(195 GB used, 1561 GB avail, 1830 GB total, peers []/[])
2013-02-13 19:39:26.477504 7f76717b9700  5 osd.0 10 heartbeat: osd_stat(195 GB used, 1561 GB avail, 1830 GB total, peers []/[])
2013-02-13 19:39:26.477538 7f76717b9700 10 osd.0 10 i have no heartbeat peers; checking mon for new map
2013-02-13 19:39:27.230077 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:27.230096 7f76807d7700 20 osd.0 10 scrub_random_backoff lost coin flip, randomly backing off
2013-02-13 19:39:28.230166 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:28.230186 7f76807d7700 20 osd.0 10 scrub_random_backoff lost coin flip, randomly backing off
2013-02-13 19:39:29.230264 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:29.230313 7f76807d7700 20 osd.0 10 scrub_should_schedule loadavg 0.01 < max 0.5 = yes
2013-02-13 19:39:29.230329 7f76807d7700 20 osd.0 10 sched_scrub load_is_low=1
2013-02-13 19:39:29.231936 7f76807d7700 20 osd.0 10 sched_scrub done
2013-02-13 19:39:29.231939 7f76807d7700  7 osd.0 10 do_mon_report
2013-02-13 19:39:29.231942 7f76807d7700 10 osd.0 10 send_alive up_thru currently 7 want 7
2013-02-13 19:39:29.231945 7f76807d7700 20 osd.0 10 send_pg_stats
2013-02-13 19:39:30.232019 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:30.232064 7f76807d7700 20 osd.0 10 scrub_should_schedule loadavg 0.01 < max 0.5 = yes
2013-02-13 19:39:30.232078 7f76807d7700 20 osd.0 10 sched_scrub load_is_low=1
2013-02-13 19:39:30.233066 7f76807d7700 20 osd.0 10 sched_scrub done
2013-02-13 19:39:31.233138 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:31.233156 7f76807d7700 20 osd.0 10 scrub_random_backoff lost coin flip, randomly backing off
2013-02-13 19:39:32.233233 7f76807d7700  5 osd.0 10 tick
2013-02-13 19:39:32.233278 7f76807d7700 20 osd.0 10 scrub_should_schedule loadavg 0.01 < max 0.5 = yes





-- Test command

2013-02-13 19:39:25.463912 7ff4d1b94780  1 -- :/0 messenger.start
2013-02-13 19:39:25.464010 7ff4d1b94780  5 adding auth protocol: cephx
2013-02-13 19:39:25.464193 7ff4d1b94780  2 auth: KeyRing::load: loaded key file /etc/ceph/ceph.keyring
2013-02-13 19:39:25.464481 7ff4d1b94780  1 -- :/1004293 --> 10.x.y.z:6789/0 -- auth(proto 0 32 bytes epoch 0) v1 -- ?+0 0x1ff4640 con 0x1ff4250
2013-02-13 19:39:25.464792 7ff4d1b8c700  1 -- 10.x.y.z:0/1004293 learned my addr 10.x.y.z:0/1004293
2013-02-13 19:39:25.465515 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== mon.0 10.x.y.z:6789/0 1 ==== mon_map v1 ==== 191+0+0 (203010237 0 0) 0x7ff4b80009f0 con 0x1ff4250
2013-02-13 19:39:25.465642 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== mon.0 10.x.y.z:6789/0 2 ==== auth_reply(proto 2 0 Success) v1 ==== 33+0+0 (1883978408 0 0) 0x7ff4b8000d90 con 0x1ff4250
2013-02-13 19:39:25.465678 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service mon
2013-02-13 19:39:25.465681 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service osd
2013-02-13 19:39:25.465682 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service auth
2013-02-13 19:39:25.465683 7ff4cd9f5700 10 cephx: validate_tickets want 37 have 0 need 37
2013-02-13 19:39:25.465691 7ff4cd9f5700 10 cephx client: handle_response ret = 0
2013-02-13 19:39:25.465692 7ff4cd9f5700 10 cephx client:  got initial server challenge 3799155825210509375
2013-02-13 19:39:25.465698 7ff4cd9f5700 10 cephx client: build_request
2013-02-13 19:39:25.465699 7ff4cd9f5700 10 cephx client: validate_tickets: want=37 need=37 have=0
2013-02-13 19:39:25.465700 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service mon
2013-02-13 19:39:25.465701 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service osd
2013-02-13 19:39:25.465702 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service auth
2013-02-13 19:39:25.465703 7ff4cd9f5700 10 cephx: validate_tickets want 37 have 0 need 37
2013-02-13 19:39:25.465704 7ff4cd9f5700 10 cephx client: want=37 need=37 have=0
2013-02-13 19:39:25.465854 7ff4cd9f5700 10 cephx client: get auth session key: client_challenge 129993988519795561
2013-02-13 19:39:25.465859 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 --> 10.x.y.z:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7ff4bc0017a0 con 0x1ff4250
2013-02-13 19:39:25.466159 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== mon.0 10.x.y.z:6789/0 3 ==== auth_reply(proto 2 0 Success) v1 ==== 222+0+0 (1281992791 0 0) 0x7ff4b8000d90 con 0x1ff4250
2013-02-13 19:39:25.466173 7ff4cd9f5700 10 cephx client: handle_response ret = 0
2013-02-13 19:39:25.466175 7ff4cd9f5700 10 cephx client:  get_auth_session_key
2013-02-13 19:39:25.466180 7ff4cd9f5700 10 cephx: verify_service_ticket_reply got 1 keys
2013-02-13 19:39:25.466181 7ff4cd9f5700 10 cephx: got key for service_id auth
2013-02-13 19:39:25.466209 7ff4cd9f5700 10 cephx:  ticket.secret_id=2
2013-02-13 19:39:25.466211 7ff4cd9f5700 10 cephx: verify_service_ticket_reply service auth secret_id 2 session_key AQBtXBxR0N7GGxAAls9i2W1MOixZq7kA3/i4EQ== validity=43200.000000
2013-02-13 19:39:25.466218 7ff4cd9f5700 10 cephx: ticket expires=2013-02-14 07:39:25.466217 renew_after=2013-02-14 04:39:25.466217
2013-02-13 19:39:25.466222 7ff4cd9f5700 10 cephx client:  want=37 need=37 have=0
2013-02-13 19:39:25.466224 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service mon
2013-02-13 19:39:25.466224 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service osd
2013-02-13 19:39:25.466227 7ff4cd9f5700 10 cephx: validate_tickets want 37 have 32 need 5
2013-02-13 19:39:25.466228 7ff4cd9f5700 10 cephx client: build_request
2013-02-13 19:39:25.466228 7ff4cd9f5700 10 cephx client: validate_tickets: want=37 need=5 have=32
2013-02-13 19:39:25.466229 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service mon
2013-02-13 19:39:25.466230 7ff4cd9f5700 10 cephx: set_have_need_key no handler for service osd
2013-02-13 19:39:25.466230 7ff4cd9f5700 10 cephx: validate_tickets want 37 have 32 need 5
2013-02-13 19:39:25.466231 7ff4cd9f5700 10 cephx client: want=37 need=5 have=32
2013-02-13 19:39:25.466231 7ff4cd9f5700 10 cephx client: get service keys: want=37 need=5 have=32
2013-02-13 19:39:25.466252 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 --> 10.x.y.z:6789/0 -- auth(proto 2 181 bytes epoch 0) v1 -- ?+0 0x7ff4bc003db0 con 0x1ff4250
2013-02-13 19:39:25.466499 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== mon.0 10.x.y.z:6789/0 4 ==== auth_reply(proto 2 0 Success) v1 ==== 489+0+0 (1543017434 0 0) 0x7ff4b8000fc0 con 0x1ff4250
2013-02-13 19:39:25.466526 7ff4cd9f5700 10 cephx client: handle_response ret = 0
2013-02-13 19:39:25.466530 7ff4cd9f5700 10 cephx client:  get_principal_session_key session_key AQBtXBxR0N7GGxAAls9i2W1MOixZq7kA3/i4EQ==
2013-02-13 19:39:25.466545 7ff4cd9f5700 10 cephx: verify_service_ticket_reply got 2 keys
2013-02-13 19:39:25.466548 7ff4cd9f5700 10 cephx: got key for service_id mon
2013-02-13 19:39:25.466575 7ff4cd9f5700 10 cephx:  ticket.secret_id=24
2013-02-13 19:39:25.466578 7ff4cd9f5700 10 cephx: verify_service_ticket_reply service mon secret_id 24 session_key AQBtXBxReJ/MGxAAkdTyaziy7g4lTE185BRwQQ== validity=3600.000000
2013-02-13 19:39:25.466589 7ff4cd9f5700 10 cephx: ticket expires=2013-02-13 20:39:25.466588 renew_after=2013-02-13 20:24:25.466588
2013-02-13 19:39:25.466596 7ff4cd9f5700 10 cephx: got key for service_id osd
2013-02-13 19:39:25.466612 7ff4cd9f5700 10 cephx:  ticket.secret_id=24
2013-02-13 19:39:25.466615 7ff4cd9f5700 10 cephx: verify_service_ticket_reply service osd secret_id 24 session_key AQBtXBxR0LrMGxAAkusOMo2h5js9keN/Xr14SA== validity=3600.000000
2013-02-13 19:39:25.466623 7ff4cd9f5700 10 cephx: ticket expires=2013-02-13 20:39:25.466623 renew_after=2013-02-13 20:24:25.466623
2013-02-13 19:39:25.466629 7ff4cd9f5700 10 cephx: validate_tickets want 37 have 37 need 0
2013-02-13 19:39:25.466637 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 --> 10.x.y.z:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x1ff4b60 con 0x1ff4250
2013-02-13 19:39:25.466666 7ff4cd9f5700 10 cephx: validate_tickets want 37 have 37 need 0
2013-02-13 19:39:25.466668 7ff4cd9f5700 20 cephx client: need_tickets: want=37 need=0 have=37
2013-02-13 19:39:25.466754 7ff4d1b94780  1 -- 10.x.y.z:0/1004293 --> 10.x.y.z:6789/0 -- mon_subscribe({monmap=2+,osdmap=0}) v2 -- ?+0 0x1ff59e0 con 0x1ff4250
2013-02-13 19:39:25.466762 7ff4d1b94780  1 -- 10.x.y.z:0/1004293 --> 10.x.y.z:6789/0 -- mon_subscribe({monmap=2+,osdmap=0}) v2 -- ?+0 0x1ff5f30 con 0x1ff4250
2013-02-13 19:39:25.467034 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== mon.0 10.x.y.z:6789/0 5 ==== mon_map v1 ==== 191+0+0 (203010237 0 0) 0x7ff4b8000f90 con 0x1ff4250
2013-02-13 19:39:25.467078 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== mon.0 10.x.y.z:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (676355301 0 0) 0x7ff4b8001170 con 0x1ff4250
2013-02-13 19:39:25.467165 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== mon.0 10.x.y.z:6789/0 7 ==== osd_map(10..10 src has 1..10) v3 ==== 1513+0+0 (3135211064 0 0) 0x7ff4b8000ae0 con 0x1ff4250
2013-02-13 19:39:25.467364 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== mon.0 10.x.y.z:6789/0 8 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (676355301 0 0) 0x7ff4b80016d0 con 0x1ff4250
2013-02-13 19:39:25.467590 7ff4d1b94780  1 -- 10.x.y.z:0/1004293 --> 10.x.y.z:6801/400 -- osd_op(client.4254.0:1 rbd_directory [read 0~0] 3.30a98c1c) v4 -- ?+0 0x1ff6b70 con 0x1ff68b0
2013-02-13 19:39:25.467792 7ff4c67fc700 10 cephx client: build_authorizer for service osd
2013-02-13 19:39:25.467962 7ff4c67fc700 10 In get_auth_session_handler for protocol 2
2013-02-13 19:39:25.467996 7ff4c67fc700 10 sign_message: seq # 1 CRCs are: header 1763447648 front 1934294822 middle 0 data 0
2013-02-13 19:39:25.468037 7ff4c67fc700 20 Putting signature in client message(seq # 1): sig = 2520484184978126828
2013-02-13 19:39:25.468442 7ff4c66fb700 10 check_message_signature: seq # = 1 front_crc_ = 823660382 middle_crc = 0 data_crc = 0
2013-02-13 19:39:25.468531 7ff4cd9f5700  1 -- 10.x.y.z:0/1004293 <== osd.0 10.x.y.z:6801/400 1 ==== osd_op_reply(1 rbd_directory [read 0~0] = -1 (Operation not permitted)) v4 ==== 112+0+0 (823660382 0 0) 0x7ff4b40009a0 con 0x1ff68b0
rbd: list: (1) Operation not permitted
2013-02-13 19:39:25.468952 7ff4d1b94780  1 -- 10.x.y.z:0/1004293 mark_down 0x1ff68b0 -- 0x1ff6660
2013-02-13 19:39:25.469112 7ff4d1b94780  1 -- 10.x.y.z:0/1004293 mark_down_all
2013-02-13 19:39:25.469220 7ff4d1b94780  1 -- 10.x.y.z:0/1004293 shutdown complete.