API tools faq
paste
alohamora

storage0113

alohamora
May 21st, 2014
438
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 71.05 KB | None | 0 0
raw download clone embed print report
  1. [root@storage0113-ib ~]# service ceph status
  2. === osd.163 ===
  3. osd.163: running {"version":"0.80-rc1-355-gd34cc1e"}
  4. === osd.139 ===
  5. osd.139: dead.
  6. === osd.78 ===
  7. osd.78: dead.
  8. === osd.152 ===
  9. osd.152: dead.
  10. === osd.128 ===
  11. osd.128: dead.
  12. === osd.116 ===
  13. osd.116: dead.
  14. === osd.35 ===
  15. osd.35: dead.
  16. === osd.104 ===
  17. osd.104: dead.
  18. === osd.89 ===
  19. osd.89: dead.
  20. === osd.55 ===
  21. osd.55: dead.
  22. === osd.67 ===
  23. osd.67: dead.
  24. [root@storage0113-ib ~]#
  25. [root@storage0113-ib ~]# ps -ef | grep -i osd
  26. root 1263 1 2 14:52 ? 00:00:36 /usr/bin/ceph-osd -i 163 --pid-file /var/run/ceph/osd.163.pid -c /etc/ceph/ceph.conf --cluster ceph
  27. root 7096 6329 0 15:16 pts/1 00:00:00 grep -i osd
  28. [root@storage0113-ib ~]#
  29. [root@storage0113-ib ~]#
  30. [root@storage0113-ib ~]#
  31. [root@storage0113-ib ~]# service ceph stats
  32. === osd.163 ===
  33. usage: /etc/init.d/ceph [options] {start|stop|restart|condrestart} [mon|osd|mds]...
  34. -c ceph.conf
  35. --valgrind run via valgrind
  36. --hostname [hostname] override hostname lookup
  37. [root@storage0113-ib ~]# service ceph start
  38. === osd.163 ===
  39. Starting Ceph osd.163 on storage0113-ib...already running
  40. === osd.139 ===
  41. 2014-05-21 15:16:13.700777 7f7bfb9df700 1 -- :/0 messenger.start
  42. 2014-05-21 15:16:13.702155 7f7bfb9df700 1 -- :/1007419 --> 192.168.100.101:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- ?+0 0x7f7bf4022b00 con 0x7f7bf4022690
  43. 2014-05-21 15:16:13.702843 7f7bf21fc700 1 -- 192.168.100.113:0/1007419 learned my addr 192.168.100.113:0/1007419
  44. 2014-05-21 15:16:13.703982 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f7be8000c90 con 0x7f7bf4022690
  45. 2014-05-21 15:16:13.704164 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (3788151119 0 0) 0x7f7be8001180 con 0x7f7bf4022690
  46. 2014-05-21 15:16:13.704430 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f7bdc001970 con 0x7f7bf4022690
  47. 2014-05-21 15:16:13.705372 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (3063146698 0 0) 0x7f7be8001180 con 0x7f7bf4022690
  48. 2014-05-21 15:16:13.705536 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 --> 192.168.100.101:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7f7bdc002280 con 0x7f7bf4022690
  49. 2014-05-21 15:16:13.706240 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (3539765734 0 0) 0x7f7be80008c0 con 0x7f7bf4022690
  50. 2014-05-21 15:16:13.706369 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7f7bf4022f50 con 0x7f7bf4022690
  51. 2014-05-21 15:16:13.706429 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 --> 192.168.100.101:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7f7bdc002de0 con 0x7f7bf4022690
  52. 2014-05-21 15:16:13.706656 7f7bfb9df700 1 -- 192.168.100.113:0/1007419 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f7bf4022b00 con 0x7f7bf4022690
  53. 2014-05-21 15:16:13.706686 7f7bfb9df700 1 -- 192.168.100.113:0/1007419 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f7bf4023fc0 con 0x7f7bf4022690
  54. 2014-05-21 15:16:13.706903 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f7be8001470 con 0x7f7bf4022690
  55. 2014-05-21 15:16:13.706984 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f7be8001710 con 0x7f7bf4022690
  56. 2014-05-21 15:16:13.707073 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (983183946 0 0) 0x7f7be80008c0 con 0x7f7bf4022690
  57. 2014-05-21 15:16:13.709622 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 8 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f7be80008c0 con 0x7f7bf4022690
  58. 2014-05-21 15:16:13.712656 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f7be802ae20 con 0x7f7bf4022690
  59. 2014-05-21 15:16:13.712694 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 10 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f7be8054b70 con 0x7f7bf4022690
  60. 2014-05-21 15:16:13.712716 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f7be8054fe0 con 0x7f7bf4022690
  61. 2014-05-21 15:16:13.714031 7f7bfb9df700 1 -- 192.168.100.113:0/1007419 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7f7bf40172b0 con 0x7f7bf4022690
  62. 2014-05-21 15:16:13.718635 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7f7be80018b0 con 0x7f7bf4022690
  63. 2014-05-21 15:16:13.941318 7f7bfb9df700 1 -- 192.168.100.113:0/1007419 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 139, "weight": 2.73} v 0) v1 -- ?+0 0x7f7bf4022b00 con 0x7f7bf4022690
  64. 2014-05-21 15:16:13.943562 7f7bf94a2700 1 -- 192.168.100.113:0/1007419 <== mon.0 192.168.100.101:6789/0 13 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 139, "weight": 2.73}]=0 create-or-move updated item name 'osd.139' weight 2.73 at location {host=storage0113-ib,root=default} to crush map v358031) v1 ==== 262+0+0 (665910523 0 0) 0x7f7be80008c0 con 0x7f7bf4022690
  65. create-or-move updated item name 'osd.139' weight 2.73 at location {host=storage0113-ib,root=default} to crush map
  66. 2014-05-21 15:16:13.949380 7f7bfb9df700 1 -- 192.168.100.113:0/1007419 mark_down 0x7f7bf4022690 -- 0x7f7bf4022420
  67. 2014-05-21 15:16:13.949450 7f7bfb9df700 1 -- 192.168.100.113:0/1007419 mark_down_all
  68. 2014-05-21 15:16:13.950175 7f7bfb9df700 1 -- 192.168.100.113:0/1007419 shutdown complete.
  69. Starting Ceph osd.139 on storage0113-ib...
  70. starting osd.139 at :/0 osd_data /var/lib/ceph/osd/ceph-139 /var/lib/ceph/osd/ceph-139/journal
  71. === osd.78 ===
  72. 2014-05-21 15:16:14.530335 7f8c9533e700 1 -- :/0 messenger.start
  73. 2014-05-21 15:16:14.531526 7f8c9533e700 1 -- :/1007600 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7f8c90022b00 con 0x7f8c90022690
  74. 2014-05-21 15:16:14.531932 7f8c94203700 1 -- 192.168.100.113:0/1007600 learned my addr 192.168.100.113:0/1007600
  75. 2014-05-21 15:16:14.532892 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f8c78000c90 con 0x7f8c90022690
  76. 2014-05-21 15:16:14.533068 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2656032508 0 0) 0x7f8c78001140 con 0x7f8c90022690
  77. 2014-05-21 15:16:14.533325 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f8c7c001970 con 0x7f8c90022690
  78. 2014-05-21 15:16:14.534566 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (266836519 0 0) 0x7f8c78001140 con 0x7f8c90022690
  79. 2014-05-21 15:16:14.534785 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 --> 192.168.100.101:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7f8c7c002280 con 0x7f8c90022690
  80. 2014-05-21 15:16:14.535789 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (1374981663 0 0) 0x7f8c780008c0 con 0x7f8c90022690
  81. 2014-05-21 15:16:14.535970 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7f8c90022f50 con 0x7f8c90022690
  82. 2014-05-21 15:16:14.536030 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 --> 192.168.100.101:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7f8c7c002de0 con 0x7f8c90022690
  83. 2014-05-21 15:16:14.536309 7f8c9533e700 1 -- 192.168.100.113:0/1007600 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f8c90022b00 con 0x7f8c90022690
  84. 2014-05-21 15:16:14.536340 7f8c9533e700 1 -- 192.168.100.113:0/1007600 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f8c90023ef0 con 0x7f8c90022690
  85. 2014-05-21 15:16:14.536557 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f8c780013b0 con 0x7f8c90022690
  86. 2014-05-21 15:16:14.536649 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f8c78001650 con 0x7f8c90022690
  87. 2014-05-21 15:16:14.536681 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (2042673164 0 0) 0x7f8c780008c0 con 0x7f8c90022690
  88. 2014-05-21 15:16:14.539108 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 8 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f8c780008c0 con 0x7f8c90022690
  89. 2014-05-21 15:16:14.542113 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f8c7802ae30 con 0x7f8c90022690
  90. 2014-05-21 15:16:14.542150 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 10 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f8c78054b80 con 0x7f8c90022690
  91. 2014-05-21 15:16:14.542172 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f8c78054ff0 con 0x7f8c90022690
  92. 2014-05-21 15:16:14.543643 7f8c9533e700 1 -- 192.168.100.113:0/1007600 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7f8c900172b0 con 0x7f8c90022690
  93. 2014-05-21 15:16:14.547723 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7f8c780017a0 con 0x7f8c90022690
  94. 2014-05-21 15:16:14.697510 7f8c9533e700 1 -- 192.168.100.113:0/1007600 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 78, "weight": 2.73} v 0) v1 -- ?+0 0x7f8c900172b0 con 0x7f8c90022690
  95. 2014-05-21 15:16:14.699610 7f8c8ebfd700 1 -- 192.168.100.113:0/1007600 <== mon.0 192.168.100.101:6789/0 13 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 78, "weight": 2.73}]=0 create-or-move updated item name 'osd.78' weight 2.73 at location {host=storage0113-ib,root=default} to crush map v358031) v1 ==== 260+0+0 (3353423989 0 0) 0x7f8c780009d0 con 0x7f8c90022690
  96. create-or-move updated item name 'osd.78' weight 2.73 at location {host=storage0113-ib,root=default} to crush map
  97. 2014-05-21 15:16:14.706421 7f8c9533e700 1 -- 192.168.100.113:0/1007600 mark_down 0x7f8c90022690 -- 0x7f8c90022420
  98. 2014-05-21 15:16:14.706476 7f8c9533e700 1 -- 192.168.100.113:0/1007600 mark_down_all
  99. 2014-05-21 15:16:14.706645 7f8c9533e700 1 -- 192.168.100.113:0/1007600 shutdown complete.
  100. Starting Ceph osd.78 on storage0113-ib...
  101. starting osd.78 at :/0 osd_data /var/lib/ceph/osd/ceph-78 /var/lib/ceph/osd/ceph-78/journal
  102. === osd.152 ===
  103. 2014-05-21 15:16:15.265336 7fd908e88700 1 -- :/0 messenger.start
  104. 2014-05-21 15:16:15.274085 7fd908e88700 1 -- :/1007779 --> 192.168.100.101:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- ?+0 0x7fd904022b00 con 0x7fd904022690
  105. 2014-05-21 15:16:15.274538 7fd908248700 1 -- 192.168.100.113:0/1007779 learned my addr 192.168.100.113:0/1007779
  106. 2014-05-21 15:16:15.275705 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7fd8ec000c90 con 0x7fd904022690
  107. 2014-05-21 15:16:15.275878 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (98406670 0 0) 0x7fd8ec001140 con 0x7fd904022690
  108. 2014-05-21 15:16:15.276134 7fd902702700 1 -- 192.168.100.113:0/1007779 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7fd8f0001970 con 0x7fd904022690
  109. 2014-05-21 15:16:15.276989 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (2594964392 0 0) 0x7fd8ec001140 con 0x7fd904022690
  110. 2014-05-21 15:16:15.277186 7fd902702700 1 -- 192.168.100.113:0/1007779 --> 192.168.100.101:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7fd8f0002280 con 0x7fd904022690
  111. 2014-05-21 15:16:15.278189 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (1797626641 0 0) 0x7fd8ec0008c0 con 0x7fd904022690
  112. 2014-05-21 15:16:15.278339 7fd902702700 1 -- 192.168.100.113:0/1007779 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7fd904022f50 con 0x7fd904022690
  113. 2014-05-21 15:16:15.278407 7fd902702700 1 -- 192.168.100.113:0/1007779 --> 192.168.100.101:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7fd8f0002de0 con 0x7fd904022690
  114. 2014-05-21 15:16:15.278666 7fd908e88700 1 -- 192.168.100.113:0/1007779 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7fd904022b00 con 0x7fd904022690
  115. 2014-05-21 15:16:15.278722 7fd908e88700 1 -- 192.168.100.113:0/1007779 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7fd904023ef0 con 0x7fd904022690
  116. 2014-05-21 15:16:15.278925 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7fd8ec0013b0 con 0x7fd904022690
  117. 2014-05-21 15:16:15.279035 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fd8ec001650 con 0x7fd904022690
  118. 2014-05-21 15:16:15.279243 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (2456036098 0 0) 0x7fd8ec0013b0 con 0x7fd904022690
  119. 2014-05-21 15:16:15.281662 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 8 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7fd8ec000970 con 0x7fd904022690
  120. 2014-05-21 15:16:15.284735 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fd8ec02ae00 con 0x7fd904022690
  121. 2014-05-21 15:16:15.284778 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 10 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7fd8ec054b50 con 0x7fd904022690
  122. 2014-05-21 15:16:15.284801 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fd8ec054fc0 con 0x7fd904022690
  123. 2014-05-21 15:16:15.294673 7fd908e88700 1 -- 192.168.100.113:0/1007779 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7fd9040172b0 con 0x7fd904022690
  124. 2014-05-21 15:16:15.298873 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7fd8ec0017a0 con 0x7fd904022690
  125. 2014-05-21 15:16:15.489227 7fd908e88700 1 -- 192.168.100.113:0/1007779 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 152, "weight": 2.73} v 0) v1 -- ?+0 0x7fd904023ef0 con 0x7fd904022690
  126. 2014-05-21 15:16:15.490856 7fd902702700 1 -- 192.168.100.113:0/1007779 <== mon.0 192.168.100.101:6789/0 13 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 152, "weight": 2.73}]=0 create-or-move updated item name 'osd.152' weight 2.73 at location {host=storage0113-ib,root=default} to crush map v358031) v1 ==== 262+0+0 (3559762588 0 0) 0x7fd8ec0009d0 con 0x7fd904022690
  127. create-or-move updated item name 'osd.152' weight 2.73 at location {host=storage0113-ib,root=default} to crush map
  128. 2014-05-21 15:16:15.497748 7fd908e88700 1 -- 192.168.100.113:0/1007779 mark_down 0x7fd904022690 -- 0x7fd904022420
  129. 2014-05-21 15:16:15.497838 7fd908e88700 1 -- 192.168.100.113:0/1007779 mark_down_all
  130. 2014-05-21 15:16:15.497915 7fd908e88700 1 -- 192.168.100.113:0/1007779 shutdown complete.
  131. Starting Ceph osd.152 on storage0113-ib...
  132. starting osd.152 at :/0 osd_data /var/lib/ceph/osd/ceph-152 /var/lib/ceph/osd/ceph-152/journal
  133. === osd.128 ===
  134. 2014-05-21 15:16:16.077437 7f9026b69700 1 -- :/0 messenger.start
  135. 2014-05-21 15:16:16.091855 7f9026b69700 1 -- :/1007928 --> 192.168.100.101:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- ?+0 0x7f9020022b00 con 0x7f9020022690
  136. 2014-05-21 15:16:16.092352 7f902462c700 1 -- 192.168.100.113:0/1007928 learned my addr 192.168.100.113:0/1007928
  137. 2014-05-21 15:16:16.093260 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f9014000c90 con 0x7f9020022690
  138. 2014-05-21 15:16:16.093460 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (895745588 0 0) 0x7f9014001140 con 0x7f9020022690
  139. 2014-05-21 15:16:16.093707 7f901ffff700 1 -- 192.168.100.113:0/1007928 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f9008001970 con 0x7f9020022690
  140. 2014-05-21 15:16:16.094494 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (255784917 0 0) 0x7f9014001140 con 0x7f9020022690
  141. 2014-05-21 15:16:16.094706 7f901ffff700 1 -- 192.168.100.113:0/1007928 --> 192.168.100.101:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7f9008002280 con 0x7f9020022690
  142. 2014-05-21 15:16:16.095592 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (3828572736 0 0) 0x7f90140008c0 con 0x7f9020022690
  143. 2014-05-21 15:16:16.095770 7f901ffff700 1 -- 192.168.100.113:0/1007928 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7f9020022f50 con 0x7f9020022690
  144. 2014-05-21 15:16:16.095819 7f901ffff700 1 -- 192.168.100.113:0/1007928 --> 192.168.100.101:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7f9008002de0 con 0x7f9020022690
  145. 2014-05-21 15:16:16.096042 7f9026b69700 1 -- 192.168.100.113:0/1007928 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f9020022b00 con 0x7f9020022690
  146. 2014-05-21 15:16:16.096072 7f9026b69700 1 -- 192.168.100.113:0/1007928 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f9020023ef0 con 0x7f9020022690
  147. 2014-05-21 15:16:16.096232 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f90140013b0 con 0x7f9020022690
  148. 2014-05-21 15:16:16.096341 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f9014001650 con 0x7f9020022690
  149. 2014-05-21 15:16:16.096393 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (4088407749 0 0) 0x7f90140008c0 con 0x7f9020022690
  150. 2014-05-21 15:16:16.098688 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 8 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f90140008c0 con 0x7f9020022690
  151. 2014-05-21 15:16:16.101557 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f901402ae30 con 0x7f9020022690
  152. 2014-05-21 15:16:16.101593 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 10 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f9014054b80 con 0x7f9020022690
  153. 2014-05-21 15:16:16.101615 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f9014054ff0 con 0x7f9020022690
  154. 2014-05-21 15:16:16.106225 7f9026b69700 1 -- 192.168.100.113:0/1007928 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7f90200172b0 con 0x7f9020022690
  155. 2014-05-21 15:16:16.110997 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7f90140017a0 con 0x7f9020022690
  156. 2014-05-21 15:16:16.326598 7f9026b69700 1 -- 192.168.100.113:0/1007928 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 128, "weight": 2.73} v 0) v1 -- ?+0 0x7f90200172b0 con 0x7f9020022690
  157. 2014-05-21 15:16:16.328475 7f901ffff700 1 -- 192.168.100.113:0/1007928 <== mon.0 192.168.100.101:6789/0 13 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 128, "weight": 2.73}]=0 create-or-move updated item name 'osd.128' weight 2.73 at location {host=storage0113-ib,root=default} to crush map v358031) v1 ==== 262+0+0 (4278246908 0 0) 0x7f90140009d0 con 0x7f9020022690
  158. create-or-move updated item name 'osd.128' weight 2.73 at location {host=storage0113-ib,root=default} to crush map
  159. 2014-05-21 15:16:16.337485 7f9026b69700 1 -- 192.168.100.113:0/1007928 mark_down 0x7f9020022690 -- 0x7f9020022420
  160. 2014-05-21 15:16:16.337539 7f9026b69700 1 -- 192.168.100.113:0/1007928 mark_down_all
  161. 2014-05-21 15:16:16.337876 7f9026b69700 1 -- 192.168.100.113:0/1007928 shutdown complete.
  162. Starting Ceph osd.128 on storage0113-ib...
  163. starting osd.128 at :/0 osd_data /var/lib/ceph/osd/ceph-128 /var/lib/ceph/osd/ceph-128/journal
  164. === osd.116 ===
  165. 2014-05-21 15:16:16.904491 7f50fc690700 1 -- :/0 messenger.start
  166. 2014-05-21 15:16:16.918731 7f50fc690700 1 -- :/1008151 --> 192.168.100.101:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- ?+0 0x7f50f4022b00 con 0x7f50f4022690
  167. 2014-05-21 15:16:16.919175 7f50f8350700 1 -- 192.168.100.113:0/1008151 learned my addr 192.168.100.113:0/1008151
  168. 2014-05-21 15:16:16.920057 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f50e0000c90 con 0x7f50f4022690
  169. 2014-05-21 15:16:16.920221 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (1888558013 0 0) 0x7f50e0001140 con 0x7f50f4022690
  170. 2014-05-21 15:16:16.920502 7f50fa153700 1 -- 192.168.100.113:0/1008151 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f50e4001970 con 0x7f50f4022690
  171. 2014-05-21 15:16:16.921865 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (3344577755 0 0) 0x7f50e0001140 con 0x7f50f4022690
  172. 2014-05-21 15:16:16.922085 7f50fa153700 1 -- 192.168.100.113:0/1008151 --> 192.168.100.101:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7f50e4002280 con 0x7f50f4022690
  173. 2014-05-21 15:16:16.923033 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (3954459161 0 0) 0x7f50e00008c0 con 0x7f50f4022690
  174. 2014-05-21 15:16:16.923214 7f50fa153700 1 -- 192.168.100.113:0/1008151 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7f50f4022f50 con 0x7f50f4022690
  175. 2014-05-21 15:16:16.923278 7f50fa153700 1 -- 192.168.100.113:0/1008151 --> 192.168.100.101:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7f50e4002de0 con 0x7f50f4022690
  176. 2014-05-21 15:16:16.923496 7f50fc690700 1 -- 192.168.100.113:0/1008151 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f50f4022b00 con 0x7f50f4022690
  177. 2014-05-21 15:16:16.923527 7f50fc690700 1 -- 192.168.100.113:0/1008151 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f50f4023ef0 con 0x7f50f4022690
  178. 2014-05-21 15:16:16.923720 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f50e00013b0 con 0x7f50f4022690
  179. 2014-05-21 15:16:16.923819 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f50e0001650 con 0x7f50f4022690
  180. 2014-05-21 15:16:16.924222 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (4103884963 0 0) 0x7f50e0000a00 con 0x7f50f4022690
  181. 2014-05-21 15:16:16.926770 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 8 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f50e00008c0 con 0x7f50f4022690
  182. 2014-05-21 15:16:16.929632 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f50e002ae30 con 0x7f50f4022690
  183. 2014-05-21 15:16:16.929668 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 10 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f50e0054b80 con 0x7f50f4022690
  184. 2014-05-21 15:16:16.929689 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f50e0054ff0 con 0x7f50f4022690
  185. 2014-05-21 15:16:16.933932 7f50fc690700 1 -- 192.168.100.113:0/1008151 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7f50f40172b0 con 0x7f50f4022690
  186. 2014-05-21 15:16:16.938192 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7f50e00017a0 con 0x7f50f4022690
  187. 2014-05-21 15:16:17.167150 7f50fc690700 1 -- 192.168.100.113:0/1008151 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 116, "weight": 2.73} v 0) v1 -- ?+0 0x7f50f40243e0 con 0x7f50f4022690
  188. 2014-05-21 15:16:17.168496 7f50fa153700 1 -- 192.168.100.113:0/1008151 <== mon.0 192.168.100.101:6789/0 13 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 116, "weight": 2.73}]=0 create-or-move updated item name 'osd.116' weight 2.73 at location {host=storage0113-ib,root=default} to crush map v358031) v1 ==== 262+0+0 (3101275539 0 0) 0x7f50e00009d0 con 0x7f50f4022690
  189. create-or-move updated item name 'osd.116' weight 2.73 at location {host=storage0113-ib,root=default} to crush map
  190. 2014-05-21 15:16:17.178892 7f50fc690700 1 -- 192.168.100.113:0/1008151 mark_down 0x7f50f4022690 -- 0x7f50f4022420
  191. 2014-05-21 15:16:17.178989 7f50fc690700 1 -- 192.168.100.113:0/1008151 mark_down_all
  192. 2014-05-21 15:16:17.179129 7f50fc690700 1 -- 192.168.100.113:0/1008151 shutdown complete.
  193. Starting Ceph osd.116 on storage0113-ib...
  194. starting osd.116 at :/0 osd_data /var/lib/ceph/osd/ceph-116 /var/lib/ceph/osd/ceph-116/journal
  195. === osd.35 ===
  196. 2014-05-21 15:16:17.749999 7fc58790e700 1 -- :/0 messenger.start
  197. 2014-05-21 15:16:17.770025 7fc58790e700 1 -- :/1008332 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7fc580022b00 con 0x7fc580022690
  198. 2014-05-21 15:16:17.770473 7fc5849d0700 1 -- 192.168.100.113:0/1008332 learned my addr 192.168.100.113:0/1008332
  199. 2014-05-21 15:16:17.771492 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7fc574000c90 con 0x7fc580022690
  200. 2014-05-21 15:16:17.771815 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (1959197302 0 0) 0x7fc574001140 con 0x7fc580022690
  201. 2014-05-21 15:16:17.772079 7fc5853d1700 1 -- 192.168.100.113:0/1008332 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7fc568001970 con 0x7fc580022690
  202. 2014-05-21 15:16:17.772900 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (1933184511 0 0) 0x7fc574001140 con 0x7fc580022690
  203. 2014-05-21 15:16:17.773080 7fc5853d1700 1 -- 192.168.100.113:0/1008332 --> 192.168.100.101:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7fc568002280 con 0x7fc580022690
  204. 2014-05-21 15:16:17.774133 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (2159141446 0 0) 0x7fc5740008c0 con 0x7fc580022690
  205. 2014-05-21 15:16:17.774303 7fc5853d1700 1 -- 192.168.100.113:0/1008332 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7fc580022f50 con 0x7fc580022690
  206. 2014-05-21 15:16:17.774358 7fc5853d1700 1 -- 192.168.100.113:0/1008332 --> 192.168.100.101:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7fc568002de0 con 0x7fc580022690
  207. 2014-05-21 15:16:17.774803 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7fc5740013b0 con 0x7fc580022690
  208. 2014-05-21 15:16:17.777382 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fc5740008c0 con 0x7fc580022690
  209. 2014-05-21 15:16:17.777445 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (211957265 0 0) 0x7fc574001460 con 0x7fc580022690
  210. 2014-05-21 15:16:17.777727 7fc58790e700 1 -- 192.168.100.113:0/1008332 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7fc580022b00 con 0x7fc580022690
  211. 2014-05-21 15:16:17.777807 7fc58790e700 1 -- 192.168.100.113:0/1008332 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7fc580023ef0 con 0x7fc580022690
  212. 2014-05-21 15:16:17.780434 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 8 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7fc5740008c0 con 0x7fc580022690
  213. 2014-05-21 15:16:17.783249 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fc57402adc0 con 0x7fc580022690
  214. 2014-05-21 15:16:17.783306 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 10 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7fc574054b40 con 0x7fc580022690
  215. 2014-05-21 15:16:17.783328 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fc574054fe0 con 0x7fc580022690
  216. 2014-05-21 15:16:17.810885 7fc58790e700 1 -- 192.168.100.113:0/1008332 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7fc5800172b0 con 0x7fc580022690
  217. 2014-05-21 15:16:17.815207 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7fc574001790 con 0x7fc580022690
  218. 2014-05-21 15:16:18.044829 7fc58790e700 1 -- 192.168.100.113:0/1008332 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 35, "weight": 2.6299999999999999} v 0) v1 -- ?+0 0x7fc580023ef0 con 0x7fc580022690
  219. 2014-05-21 15:16:18.046937 7fc5853d1700 1 -- 192.168.100.113:0/1008332 <== mon.0 192.168.100.101:6789/0 13 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 35, "weight": 2.6299999999999999}]=0 create-or-move updated item name 'osd.35' weight 2.63 at location {host=storage0113-ib,root=default} to crush map v358031) v1 ==== 274+0+0 (2393993026 0 0) 0x7fc5740012e0 con 0x7fc580022690
  220. create-or-move updated item name 'osd.35' weight 2.63 at location {host=storage0113-ib,root=default} to crush map
  221. 2014-05-21 15:16:18.055971 7fc58790e700 1 -- 192.168.100.113:0/1008332 mark_down 0x7fc580022690 -- 0x7fc580022420
  222. 2014-05-21 15:16:18.056051 7fc58790e700 1 -- 192.168.100.113:0/1008332 mark_down_all
  223. 2014-05-21 15:16:18.056597 7fc58790e700 1 -- 192.168.100.113:0/1008332 shutdown complete.
  224. Starting Ceph osd.35 on storage0113-ib...
  225. starting osd.35 at :/0 osd_data /var/lib/ceph/osd/ceph-35 /var/lib/ceph/osd/ceph-35/journal
  226. === osd.104 ===
  227. 2014-05-21 15:16:18.613218 7ff21bfeb700 1 -- :/0 messenger.start
  228. 2014-05-21 15:16:18.619993 7ff21bfeb700 1 -- :/1008546 --> 192.168.100.101:6789/0 -- auth(proto 0 28 bytes epoch 0) v1 -- ?+0 0x7ff214022b00 con 0x7ff214022690
  229. 2014-05-21 15:16:18.620521 7ff2186ac700 1 -- 192.168.100.113:0/1008546 learned my addr 192.168.100.113:0/1008546
  230. 2014-05-21 15:16:18.621632 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7ff208000c90 con 0x7ff214022690
  231. 2014-05-21 15:16:18.621876 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (856358007 0 0) 0x7ff208001140 con 0x7ff214022690
  232. 2014-05-21 15:16:18.622132 7ff219aae700 1 -- 192.168.100.113:0/1008546 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7ff1fc001970 con 0x7ff214022690
  233. 2014-05-21 15:16:18.622932 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (3992423165 0 0) 0x7ff208001140 con 0x7ff214022690
  234. 2014-05-21 15:16:18.623116 7ff219aae700 1 -- 192.168.100.113:0/1008546 --> 192.168.100.101:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7ff1fc002280 con 0x7ff214022690
  235. 2014-05-21 15:16:18.623973 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (1615202099 0 0) 0x7ff2080008c0 con 0x7ff214022690
  236. 2014-05-21 15:16:18.624129 7ff219aae700 1 -- 192.168.100.113:0/1008546 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7ff214022f50 con 0x7ff214022690
  237. 2014-05-21 15:16:18.624190 7ff219aae700 1 -- 192.168.100.113:0/1008546 --> 192.168.100.101:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7ff1fc002de0 con 0x7ff214022690
  238. 2014-05-21 15:16:18.624377 7ff21bfeb700 1 -- 192.168.100.113:0/1008546 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7ff214022b00 con 0x7ff214022690
  239. 2014-05-21 15:16:18.624403 7ff21bfeb700 1 -- 192.168.100.113:0/1008546 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7ff214023ef0 con 0x7ff214022690
  240. 2014-05-21 15:16:18.624614 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7ff2080013b0 con 0x7ff214022690
  241. 2014-05-21 15:16:18.624705 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7ff208001650 con 0x7ff214022690
  242. 2014-05-21 15:16:18.625120 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (1199641417 0 0) 0x7ff208000a00 con 0x7ff214022690
  243. 2014-05-21 15:16:18.627115 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 8 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7ff2080008c0 con 0x7ff214022690
  244. 2014-05-21 15:16:18.629928 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7ff20802ae30 con 0x7ff214022690
  245. 2014-05-21 15:16:18.629965 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 10 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7ff208054b80 con 0x7ff214022690
  246. 2014-05-21 15:16:18.629986 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7ff208054ff0 con 0x7ff214022690
  247. 2014-05-21 15:16:18.642221 7ff21bfeb700 1 -- 192.168.100.113:0/1008546 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7ff2140172b0 con 0x7ff214022690
  248. 2014-05-21 15:16:18.646853 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7ff2080017a0 con 0x7ff214022690
  249. 2014-05-21 15:16:18.846931 7ff21bfeb700 1 -- 192.168.100.113:0/1008546 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 104, "weight": 2.73} v 0) v1 -- ?+0 0x7ff214023ef0 con 0x7ff214022690
  250. 2014-05-21 15:16:18.849273 7ff219aae700 1 -- 192.168.100.113:0/1008546 <== mon.0 192.168.100.101:6789/0 13 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 104, "weight": 2.73}]=0 create-or-move updated item name 'osd.104' weight 2.73 at location {host=storage0113-ib,root=default} to crush map v358031) v1 ==== 262+0+0 (4102971135 0 0) 0x7ff2080009d0 con 0x7ff214022690
  251. create-or-move updated item name 'osd.104' weight 2.73 at location {host=storage0113-ib,root=default} to crush map
  252. 2014-05-21 15:16:18.855231 7ff21bfeb700 1 -- 192.168.100.113:0/1008546 mark_down 0x7ff214022690 -- 0x7ff214022420
  253. 2014-05-21 15:16:18.855312 7ff21bfeb700 1 -- 192.168.100.113:0/1008546 mark_down_all
  254. 2014-05-21 15:16:18.855876 7ff21bfeb700 1 -- 192.168.100.113:0/1008546 shutdown complete.
  255. Starting Ceph osd.104 on storage0113-ib...
  256. starting osd.104 at :/0 osd_data /var/lib/ceph/osd/ceph-104 /var/lib/ceph/osd/ceph-104/journal
  257. === osd.89 ===
  258. 2014-05-21 15:16:19.277898 7f853ec81700 1 -- :/0 messenger.start
  259. 2014-05-21 15:16:19.295118 7f853ec81700 1 -- :/1008854 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7f8538022b00 con 0x7f8538022690
  260. 2014-05-21 15:16:19.295451 7f853c744700 1 -- 192.168.100.113:0/1008854 learned my addr 192.168.100.113:0/1008854
  261. 2014-05-21 15:16:19.296146 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f852c000c90 con 0x7f8538022690
  262. 2014-05-21 15:16:19.296426 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (669576960 0 0) 0x7f852c0008c0 con 0x7f8538022690
  263. 2014-05-21 15:16:19.296600 7f8537fff700 1 -- 192.168.100.113:0/1008854 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f8520001970 con 0x7f8538022690
  264. 2014-05-21 15:16:19.297721 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (240237021 0 0) 0x7f852c0008c0 con 0x7f8538022690
  265. 2014-05-21 15:16:19.297820 7f8537fff700 1 -- 192.168.100.113:0/1008854 --> 192.168.100.101:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7f8520002280 con 0x7f8538022690
  266. 2014-05-21 15:16:19.298525 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (1153671800 0 0) 0x7f852c001030 con 0x7f8538022690
  267. 2014-05-21 15:16:19.298624 7f8537fff700 1 -- 192.168.100.113:0/1008854 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7f8538022f50 con 0x7f8538022690
  268. 2014-05-21 15:16:19.298653 7f8537fff700 1 -- 192.168.100.113:0/1008854 --> 192.168.100.101:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7f8520002de0 con 0x7f8538022690
  269. 2014-05-21 15:16:19.298771 7f853ec81700 1 -- 192.168.100.113:0/1008854 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f8538022b00 con 0x7f8538022690
  270. 2014-05-21 15:16:19.298787 7f853ec81700 1 -- 192.168.100.113:0/1008854 --> 192.168.100.101:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f8538023ef0 con 0x7f8538022690
  271. 2014-05-21 15:16:19.299067 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f852c0011e0 con 0x7f8538022690
  272. 2014-05-21 15:16:19.299142 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f852c000f30 con 0x7f8538022690
  273. 2014-05-21 15:16:19.299408 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (156215187 0 0) 0x7f852c001000 con 0x7f8538022690
  274. 2014-05-21 15:16:19.301595 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 8 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f852c02a960 con 0x7f8538022690
  275. 2014-05-21 15:16:19.303250 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f852c02add0 con 0x7f8538022690
  276. 2014-05-21 15:16:19.303278 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 10 ==== osd_map(358031..358031 src has 350946..358031) v3 ==== 170528+0+0 (3550327965 0 0) 0x7f852c000f30 con 0x7f8538022690
  277. 2014-05-21 15:16:19.303316 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f852c02add0 con 0x7f8538022690
  278. 2014-05-21 15:16:19.307338 7f853ec81700 1 -- 192.168.100.113:0/1008854 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7f85380172b0 con 0x7f8538022690
  279. 2014-05-21 15:16:19.311436 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7f852c008520 con 0x7f8538022690
  280. 2014-05-21 15:16:19.447372 7f853ec81700 1 -- 192.168.100.113:0/1008854 --> 192.168.100.101:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 89, "weight": 2.73} v 0) v1 -- ?+0 0x7f85380172b0 con 0x7f8538022690
  281. 2014-05-21 15:16:36.682987 7f8537fff700 1 -- 192.168.100.113:0/1008854 <== mon.0 192.168.100.101:6789/0 13 ==== mon_command_ack([{"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 89, "weight": 2.73}]=0 create-or-move updated item name 'osd.89' weight 2.73 at location {host=storage0113-ib,root=default} to crush map v358033) v1 ==== 260+0+0 (1145421979 0 0) 0x7f852c001000 con 0x7f8538022690
  282. create-or-move updated item name 'osd.89' weight 2.73 at location {host=storage0113-ib,root=default} to crush map
  283. 2014-05-21 15:16:36.725880 7f853ec81700 1 -- 192.168.100.113:0/1008854 mark_down 0x7f8538022690 -- 0x7f8538022420
  284. 2014-05-21 15:16:36.725964 7f853ec81700 1 -- 192.168.100.113:0/1008854 mark_down_all
  285. 2014-05-21 15:16:36.726560 7f853ec81700 1 -- 192.168.100.113:0/1008854 shutdown complete.
  286. Starting Ceph osd.89 on storage0113-ib...
  287. starting osd.89 at :/0 osd_data /var/lib/ceph/osd/ceph-89 /var/lib/ceph/osd/ceph-89/journal
  288. === osd.55 ===
  289. 2014-05-21 15:16:37.311675 7ff205d8c700 1 -- :/0 messenger.start
  290. 2014-05-21 15:16:37.322864 7ff205d8c700 1 -- :/1010362 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7ff200022b00 con 0x7ff200022690
  291. 2014-05-21 15:16:37.323237 7ff204250700 1 -- 192.168.100.113:0/1010362 learned my addr 192.168.100.113:0/1010362
  292. 2014-05-21 15:16:40.242137 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7ff1f4000c90 con 0x7ff200022690
  293. 2014-05-21 15:16:40.242343 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (3972273663 0 0) 0x7ff1f4001140 con 0x7ff200022690
  294. 2014-05-21 15:16:40.242619 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7ff1e8001970 con 0x7ff200022690
  295. 2014-05-21 15:16:40.322834 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff200022690 -- 0x7ff200022420
  296. 2014-05-21 15:16:40.323080 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.115:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec001390 con 0x7ff1ec000e70
  297. 2014-05-21 15:16:43.323210 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff1ec000e70 -- 0x7ff1ec000c00
  298. 2014-05-21 15:16:43.323393 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.110:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec0022e0 con 0x7ff1ec002030
  299. 2014-05-21 15:16:45.053724 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 <== mon.1 192.168.100.110:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (1679320783 0 0) 0x7ff1f00008c0 con 0x7ff1ec002030
  300. 2014-05-21 15:16:45.053884 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.110:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7ff1e8003330 con 0x7ff1ec002030
  301. 2014-05-21 15:16:46.323549 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff1ec002030 -- 0x7ff1ec001dc0
  302. 2014-05-21 15:16:46.323701 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.115:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec002b80 con 0x7ff1ec002a20
  303. 2014-05-21 15:16:49.323852 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff1ec002a20 -- 0x7ff1ec000cd0
  304. 2014-05-21 15:16:49.324017 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec001f60 con 0x7ff1ec003270
  305. 2014-05-21 15:16:52.324138 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff1ec003270 -- 0x7ff1ec003000
  306. 2014-05-21 15:16:52.324279 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.114:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec003db0 con 0x7ff1ec000da0
  307. 2014-05-21 15:16:55.168597 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 <== mon.2 192.168.100.114:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (2538804120 0 0) 0x7ff1f00008c0 con 0x7ff1ec000da0
  308. 2014-05-21 15:16:55.168818 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.114:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7ff1e80034d0 con 0x7ff1ec000da0
  309. 2014-05-21 15:16:55.324409 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff1ec000da0 -- 0x7ff1ec003a70
  310. 2014-05-21 15:16:55.324517 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.110:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec0045c0 con 0x7ff1ec004340
  311. 2014-05-21 15:16:58.324655 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff1ec004340 -- 0x7ff1ec0040d0
  312. 2014-05-21 15:16:58.324811 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec005090 con 0x7ff1ec004c40
  313. 2014-05-21 15:17:00.004777 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 <== mon.0 192.168.100.101:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (1167507653 0 0) 0x7ff1f00008c0 con 0x7ff1ec004c40
  314. 2014-05-21 15:17:00.004997 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7ff1e8003830 con 0x7ff1ec004c40
  315. 2014-05-21 15:17:01.324941 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff1ec004c40 -- 0x7ff1ec003a00
  316. 2014-05-21 15:17:01.325092 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.110:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec005600 con 0x7ff1ec0054a0
  317. 2014-05-21 15:17:04.325223 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 mark_down 0x7ff1ec0054a0 -- 0x7ff1ec005230
  318. 2014-05-21 15:17:04.325383 7ff1fd7fb700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.115:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7ff1ec005fe0 con 0x7ff1ec003ad0
  319. 2014-05-21 15:17:04.568197 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 <== mon.3 192.168.100.115:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (947873720 0 0) 0x7ff1f00008c0 con 0x7ff1ec003ad0
  320. 2014-05-21 15:17:04.568440 7ff1ff5fe700 1 -- 192.168.100.113:0/1010362 --> 192.168.100.115:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7ff1e8006cc0 con 0x7ff1ec003ad0
  321. failed: 'timeout 30 /usr/bin/ceph -c /etc/ceph/ceph.conf --name=osd.55 --keyring=/var/lib/ceph/osd/ceph-55/keyring osd crush create-or-move -- 55 2.73 host=storage0113-ib root=default'
  322. [root@storage0113-ib ~]#
  323. [root@storage0113-ib ~]#
  324. [root@storage0113-ib ~]#
  325. [root@storage0113-ib ~]#
  326. [root@storage0113-ib ~]#
  327. [root@storage0113-ib ~]#
  328. [root@storage0113-ib ~]#
  329. [root@storage0113-ib ~]#
  330. [root@storage0113-ib ~]#
  331. [root@storage0113-ib ~]#
  332. [root@storage0113-ib ~]#
  333. [root@storage0113-ib ~]# service ceph status
  334. === osd.163 ===
  335. osd.163: running {"version":"0.80-rc1-355-gd34cc1e"}
  336. === osd.139 ===
  337. osd.139: running {"version":"0.80-rc1-355-gd34cc1e"}
  338. === osd.78 ===
  339. osd.78: running {"version":"0.80-rc1-355-gd34cc1e"}
  340. === osd.152 ===
  341. osd.152: running {"version":"0.80-rc1-355-gd34cc1e"}
  342. === osd.128 ===
  343. osd.128: running {"version":"0.80-rc1-355-gd34cc1e"}
  344. === osd.116 ===
  345. osd.116: running {"version":"0.80-rc1-355-gd34cc1e"}
  346. === osd.35 ===
  347. osd.35: running {"version":"0.80-rc1-355-gd34cc1e"}
  348. === osd.104 ===
  349. osd.104: running {"version":"0.80-rc1-355-gd34cc1e"}
  350. === osd.89 ===
  351. osd.89: running {"version":"0.80-rc1-355-gd34cc1e"}
  352. === osd.55 ===
  353. osd.55: dead.
  354. === osd.67 ===
  355. osd.67: dead.
  356. [root@storage0113-ib ~]#
  357. [root@storage0113-ib ~]#
  358. [root@storage0113-ib ~]#
  359. [root@storage0113-ib ~]#
  360. [root@storage0113-ib ~]#
  361. [root@storage0113-ib ~]#
  362. [root@storage0113-ib ~]#
  363. [root@storage0113-ib ~]#
  364. [root@storage0113-ib ~]#
  365. [root@storage0113-ib ~]#
  366. [root@storage0113-ib ~]#
  367. [root@storage0113-ib ~]# service ceph start osd.55
  368. === osd.55 ===
  369. 2014-05-21 15:17:31.863650 7f9871df5700 1 -- :/0 messenger.start
  370. 2014-05-21 15:17:31.864046 7f9871df5700 1 -- :/1012437 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7f986c022b00 con 0x7f986c022690
  371. 2014-05-21 15:17:31.864381 7f98702b9700 1 -- 192.168.100.113:0/1012437 learned my addr 192.168.100.113:0/1012437
  372. 2014-05-21 15:17:34.864045 7f98697fb700 1 -- 192.168.100.113:0/1012437 mark_down 0x7f986c022690 -- 0x7f986c022420
  373. 2014-05-21 15:17:34.864193 7f98697fb700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.110:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7f9860001390 con 0x7f9860000e70
  374. 2014-05-21 15:17:37.864327 7f98697fb700 1 -- 192.168.100.113:0/1012437 mark_down 0x7f9860000e70 -- 0x7f9860000c00
  375. 2014-05-21 15:17:37.864498 7f98697fb700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7f98600022e0 con 0x7f9860002030
  376. 2014-05-21 15:17:40.864626 7f98697fb700 1 -- 192.168.100.113:0/1012437 mark_down 0x7f9860002030 -- 0x7f9860001dc0
  377. 2014-05-21 15:17:40.864781 7f98697fb700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7f9860002b80 con 0x7f9860002a20
  378. 2014-05-21 15:17:42.636532 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f9854000c90 con 0x7f9860002a20
  379. 2014-05-21 15:17:42.636721 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (1904970453 0 0) 0x7f9854001140 con 0x7f9860002a20
  380. 2014-05-21 15:17:42.636989 7f986b5fe700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f9858001970 con 0x7f9860002a20
  381. 2014-05-21 15:17:42.638359 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (3845037919 0 0) 0x7f9854001140 con 0x7f9860002a20
  382. 2014-05-21 15:17:42.638553 7f986b5fe700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7f9858002280 con 0x7f9860002a20
  383. 2014-05-21 15:17:42.639349 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (4113567853 0 0) 0x7f98540008c0 con 0x7f9860002a20
  384. 2014-05-21 15:17:42.639485 7f986b5fe700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7f9860001dc0 con 0x7f9860002a20
  385. 2014-05-21 15:17:42.639546 7f986b5fe700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7f9858002de0 con 0x7f9860002a20
  386. 2014-05-21 15:17:42.640010 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f98540013b0 con 0x7f9860002a20
  387. 2014-05-21 15:17:42.640088 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f9854001650 con 0x7f9860002a20
  388. 2014-05-21 15:17:42.640169 7f9871df5700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f986c022f50 con 0x7f9860002a20
  389. 2014-05-21 15:17:42.640200 7f9871df5700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f986c022420 con 0x7f9860002a20
  390. 2014-05-21 15:17:42.640839 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (774616146 0 0) 0x7f9854000a00 con 0x7f9860002a20
  391. 2014-05-21 15:17:42.643596 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 8 ==== osd_map(358037..358037 src has 350946..358037) v3 ==== 170660+0+0 (1860787403 0 0) 0x7f98540008c0 con 0x7f9860002a20
  392. 2014-05-21 15:17:42.646448 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f985402aeb0 con 0x7f9860002a20
  393. 2014-05-21 15:17:42.646507 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 10 ==== osd_map(358037..358037 src has 350946..358037) v3 ==== 170660+0+0 (1860787403 0 0) 0x7f9854054c40 con 0x7f9860002a20
  394. 2014-05-21 15:17:42.646529 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f98540550e0 con 0x7f9860002a20
  395. 2014-05-21 15:17:42.681441 7f9871df5700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7f986c0172b0 con 0x7f9860002a20
  396. 2014-05-21 15:17:42.684839 7f986b5fe700 1 -- 192.168.100.113:0/1012437 <== mon.3 192.168.100.115:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7f98540017a0 con 0x7f9860002a20
  397. 2014-05-21 15:17:42.869036 7f9871df5700 1 -- 192.168.100.113:0/1012437 --> 192.168.100.115:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 55, "weight": 2.73} v 0) v1 -- ?+0 0x7f986c0172b0 con 0x7f9860002a20
  398. failed: 'timeout 30 /usr/bin/ceph -c /etc/ceph/ceph.conf --name=osd.55 --keyring=/var/lib/ceph/osd/ceph-55/keyring osd crush create-or-move -- 55 2.73 host=storage0113-ib root=default'
  399. [root@storage0113-ib ~]#
  400. [root@storage0113-ib ~]#
  401. [root@storage0113-ib ~]# service ceph status
  402. === osd.163 ===
  403. osd.163: running {"version":"0.80-rc1-355-gd34cc1e"}
  404. === osd.139 ===
  405. osd.139: running {"version":"0.80-rc1-355-gd34cc1e"}
  406. === osd.78 ===
  407. osd.78: running {"version":"0.80-rc1-355-gd34cc1e"}
  408. === osd.152 ===
  409. osd.152: running {"version":"0.80-rc1-355-gd34cc1e"}
  410. === osd.128 ===
  411. osd.128: running {"version":"0.80-rc1-355-gd34cc1e"}
  412. === osd.116 ===
  413. osd.116: running {"version":"0.80-rc1-355-gd34cc1e"}
  414. === osd.35 ===
  415. osd.35: running {"version":"0.80-rc1-355-gd34cc1e"}
  416. === osd.104 ===
  417. osd.104: running {"version":"0.80-rc1-355-gd34cc1e"}
  418. === osd.89 ===
  419. osd.89: running {"version":"0.80-rc1-355-gd34cc1e"}
  420. === osd.55 ===
  421. osd.55: dead.
  422. === osd.67 ===
  423. osd.67: dead.
  424. [root@storage0113-ib ~]#
  425. [root@storage0113-ib ~]#
  426. [root@storage0113-ib ~]#
  427. [root@storage0113-ib ~]#
  428. [root@storage0113-ib ~]# service ceph start osd.67
  429. === osd.67 ===
  430. 2014-05-21 15:20:08.584830 7f42531b2700 1 -- :/0 messenger.start
  431. 2014-05-21 15:20:08.601231 7f42531b2700 1 -- :/1015856 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7f424c022b00 con 0x7f424c022690
  432. 2014-05-21 15:20:08.601519 7f4250274700 1 -- 192.168.100.113:0/1015856 learned my addr 192.168.100.113:0/1015856
  433. 2014-05-21 15:20:09.824779 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.0 192.168.100.101:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7f4240000c90 con 0x7f424c022690
  434. 2014-05-21 15:20:09.825031 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.0 192.168.100.101:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (3608304993 0 0) 0x7f4240001140 con 0x7f424c022690
  435. 2014-05-21 15:20:09.825302 7f4250c75700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.101:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f4234001970 con 0x7f424c022690
  436. 2014-05-21 15:20:11.601203 7f424abfd700 1 -- 192.168.100.113:0/1015856 mark_down 0x7f424c022690 -- 0x7f424c022420
  437. 2014-05-21 15:20:11.601475 7f424abfd700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7f4238001390 con 0x7f4238000e70
  438. 2014-05-21 15:20:14.326445 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (448393007 0 0) 0x7f4240001140 con 0x7f4238000e70
  439. 2014-05-21 15:20:14.326666 7f4250c75700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f4234003330 con 0x7f4238000e70
  440. 2014-05-21 15:20:14.601631 7f424abfd700 1 -- 192.168.100.113:0/1015856 mark_down 0x7f4238000e70 -- 0x7f4238000c00
  441. 2014-05-21 15:20:14.601798 7f424abfd700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7f42380022e0 con 0x7f4238002030
  442. 2014-05-21 15:20:17.601956 7f424abfd700 1 -- 192.168.100.113:0/1015856 mark_down 0x7f4238002030 -- 0x7f4238001dc0
  443. 2014-05-21 15:20:17.602124 7f424abfd700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.110:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7f4238002480 con 0x7f4238002a30
  444. 2014-05-21 15:20:17.858430 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.1 192.168.100.110:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (1022114926 0 0) 0x7f423c0008c0 con 0x7f4238002a30
  445. 2014-05-21 15:20:17.858652 7f4250c75700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.110:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f42340034d0 con 0x7f4238002a30
  446. 2014-05-21 15:20:20.602254 7f424abfd700 1 -- 192.168.100.113:0/1015856 mark_down 0x7f4238002a30 -- 0x7f42380027c0
  447. 2014-05-21 15:20:20.602427 7f424abfd700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- auth(proto 0 27 bytes epoch 4) v1 -- ?+0 0x7f4238003410 con 0x7f42380031e0
  448. 2014-05-21 15:20:21.382332 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 1 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (3573835797 0 0) 0x7f423c0008c0 con 0x7f42380031e0
  449. 2014-05-21 15:20:21.382577 7f4250c75700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7f4234003830 con 0x7f42380031e0
  450. 2014-05-21 15:20:21.384027 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (330118218 0 0) 0x7f423c001060 con 0x7f42380031e0
  451. 2014-05-21 15:20:21.384214 7f4250c75700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7f4234001f60 con 0x7f42380031e0
  452. 2014-05-21 15:20:21.385021 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (1510992826 0 0) 0x7f423c0010b0 con 0x7f42380031e0
  453. 2014-05-21 15:20:21.385164 7f4250c75700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- mon_subscribe({monmap=5+}) v2 -- ?+0 0x7f4238003620 con 0x7f42380031e0
  454. 2014-05-21 15:20:21.385226 7f4250c75700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7f4234002870 con 0x7f42380031e0
  455. 2014-05-21 15:20:21.385441 7f42531b2700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f424c022b00 con 0x7f42380031e0
  456. 2014-05-21 15:20:21.385472 7f42531b2700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7f424c022420 con 0x7f42380031e0
  457. 2014-05-21 15:20:21.386162 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 4 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f423c001330 con 0x7f42380031e0
  458. 2014-05-21 15:20:21.388700 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 5 ==== osd_map(358042..358042 src has 350946..358042) v3 ==== 170822+0+0 (2233113680 0 0) 0x7f423c02ac00 con 0x7f42380031e0
  459. 2014-05-21 15:20:21.391534 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f423c02b150 con 0x7f42380031e0
  460. 2014-05-21 15:20:21.391570 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 7 ==== osd_map(358042..358042 src has 350946..358042) v3 ==== 170822+0+0 (2233113680 0 0) 0x7f423c054f70 con 0x7f42380031e0
  461. 2014-05-21 15:20:21.391591 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 8 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7f423c055430 con 0x7f42380031e0
  462. 2014-05-21 15:20:21.402366 7f42531b2700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7f424c0172b0 con 0x7f42380031e0
  463. 2014-05-21 15:20:21.406296 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 9 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7f423c0015e0 con 0x7f42380031e0
  464. 2014-05-21 15:20:21.603898 7f42531b2700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 67, "weight": 2.73} v 0) v1 -- ?+0 0x7f424c0172b0 con 0x7f42380031e0
  465. 2014-05-21 15:20:23.602620 7f424abfd700 1 -- 192.168.100.113:0/1015856 --> 192.168.100.114:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7f4238003410 con 0x7f42380031e0
  466. 2014-05-21 15:20:24.896857 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 10 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (3485991638 0 0) 0x7f423c001180 con 0x7f42380031e0
  467. 2014-05-21 15:20:24.898412 7f4250c75700 1 -- 192.168.100.113:0/1015856 <== mon.2 192.168.100.114:6789/0 11 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (3485991638 0 0) 0x7f423c001180 con 0x7f42380031e0
  468. failed: 'timeout 30 /usr/bin/ceph -c /etc/ceph/ceph.conf --name=osd.67 --keyring=/var/lib/ceph/osd/ceph-67/keyring osd crush create-or-move -- 67 2.73 host=storage0113-ib root=default'
  469. [root@storage0113-ib ~]#
  470. [root@storage0113-ib ~]# ceph status
  471. cluster 009d3518-e60d-4f74-a26d-c08c1976263c
  472. health HEALTH_WARN 562 pgs degraded; 5 pgs incomplete; 230 pgs peering; 41 pgs stale; 127 pgs stuck inactive; 41 pgs stuck stale; 2367 pgs stuck unclean; recovery 121/2199 objects degraded (5.503%); 34/165 in osds are down; nodown,noout flag(s) set
  473. monmap e4: 4 mons at {storage0101-ib=192.168.100.101:6789/0,storage0110-ib=192.168.100.110:6789/0,storage0114-ib=192.168.100.114:6789/0,storage0115-ib=192.168.100.115:6789/0}, election epoch 18, quorum 0,1,2,3 storage0101-ib,storage0110-ib,storage0114-ib,storage0115-ib
  474. osdmap e358043: 165 osds: 131 up, 165 in
  475. flags nodown,noout
  476. pgmap v604338: 4544 pgs, 6 pools, 4309 MB data, 733 objects
  477. 3580 GB used, 357 TB / 361 TB avail
  478. 121/2199 objects degraded (5.503%)
  479. 1 inactive
  480. 11 stale+active+clean
  481. 134 peering
  482. 210 active+degraded+remapped
  483. 2 stale+incomplete
  484. 2166 active+clean
  485. 2 incomplete
  486. 2 stale+active+degraded
  487. 21 stale+active+remapped
  488. 18 remapped
  489. 346 active+degraded
  490. 96 remapped+peering
  491. 1530 active+remapped
  492. 1 stale+remapped+incomplete
  493. 4 stale+active+degraded+remapped
  494. [root@storage0113-ib ~]#
  495. [root@storage0113-ib ~]# df -h
  496. Filesystem Size Used Avail Use% Mounted on
  497. /dev/sda1 47G 6.7G 38G 16% /
  498. tmpfs 12G 0 12G 0% /dev/shm
  499. /dev/sdg1 2.8T 28G 2.7T 1% /var/lib/ceph/osd/ceph-104
  500. /dev/sdl1 2.8T 27G 2.7T 1% /var/lib/ceph/osd/ceph-163
  501. /dev/sdd1 2.8T 28G 2.7T 1% /var/lib/ceph/osd/ceph-67
  502. /dev/sdb1 2.7T 28G 2.7T 2% /var/lib/ceph/osd/ceph-35
  503. /dev/sdk1 2.8T 27G 2.7T 1% /var/lib/ceph/osd/ceph-152
  504. /dev/sdj1 2.8T 12G 2.8T 1% /var/lib/ceph/osd/ceph-139
  505. /dev/sdc1 2.8T 28G 2.7T 1% /var/lib/ceph/osd/ceph-55
  506. /dev/sdi1 2.8T 28G 2.7T 1% /var/lib/ceph/osd/ceph-128
  507. /dev/sde1 2.8T 28G 2.7T 1% /var/lib/ceph/osd/ceph-78
  508. /dev/sdh1 2.8T 28G 2.7T 1% /var/lib/ceph/osd/ceph-116
  509. /dev/sdf1 2.8T 29G 2.7T 2% /var/lib/ceph/osd/ceph-89
  510. [root@storage0113-ib ~]#
  511.  
  512. [root@storage0113-ib ~]# service ceph start osd.67
  513. === osd.67 ===
  514. 2014-05-21 15:24:01.064663 7fd942104700 1 -- :/0 messenger.start
  515. 2014-05-21 15:24:01.065190 7fd942104700 1 -- :/1017473 --> 192.168.100.101:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7fd93c022b00 con 0x7fd93c022690
  516. 2014-05-21 15:24:01.065509 7fd9405c8700 1 -- 192.168.100.113:0/1017473 learned my addr 192.168.100.113:0/1017473
  517. 2014-05-21 15:24:04.065238 7fd9397fb700 1 -- 192.168.100.113:0/1017473 mark_down 0x7fd93c022690 -- 0x7fd93c022420
  518. 2014-05-21 15:24:04.065440 7fd9397fb700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- auth(proto 0 27 bytes epoch 0) v1 -- ?+0 0x7fd930001390 con 0x7fd930000e70
  519. 2014-05-21 15:24:06.356529 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 1 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7fd924000c90 con 0x7fd930000e70
  520. 2014-05-21 15:24:06.356646 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 2 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 33+0+0 (3473499124 0 0) 0x7fd924001140 con 0x7fd930000e70
  521. 2014-05-21 15:24:06.356829 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- auth(proto 2 32 bytes epoch 0) v1 -- ?+0 0x7fd928001970 con 0x7fd930000e70
  522. 2014-05-21 15:24:06.357847 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 3 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 206+0+0 (3689809443 0 0) 0x7fd924001140 con 0x7fd930000e70
  523. 2014-05-21 15:24:06.357956 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- auth(proto 2 165 bytes epoch 0) v1 -- ?+0 0x7fd928002280 con 0x7fd930000e70
  524. 2014-05-21 15:24:06.358561 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 393+0+0 (599245238 0 0) 0x7fd9240008c0 con 0x7fd930000e70
  525. 2014-05-21 15:24:06.358671 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- mon_subscribe({monmap=0+}) v2 -- ?+0 0x7fd930001a20 con 0x7fd930000e70
  526. 2014-05-21 15:24:06.358708 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- auth(proto 2 2 bytes epoch 0) v1 -- ?+0 0x7fd928002de0 con 0x7fd930000e70
  527. 2014-05-21 15:24:06.358964 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 5 ==== mon_map v1 ==== 666+0+0 (2031687792 0 0) 0x7fd9240013b0 con 0x7fd930000e70
  528. 2014-05-21 15:24:06.359015 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 6 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fd924001650 con 0x7fd930000e70
  529. 2014-05-21 15:24:06.359096 7fd942104700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7fd93c022f50 con 0x7fd930000e70
  530. 2014-05-21 15:24:06.359113 7fd942104700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- mon_subscribe({monmap=5+,osdmap=0}) v2 -- ?+0 0x7fd93c022420 con 0x7fd930000e70
  531. 2014-05-21 15:24:06.359134 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 7 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (3075491005 0 0) 0x7fd924000a00 con 0x7fd930000e70
  532. 2014-05-21 15:24:06.361480 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 8 ==== osd_map(358049..358049 src has 350946..358049) v3 ==== 170826+0+0 (955659184 0 0) 0x7fd9240008c0 con 0x7fd930000e70
  533. 2014-05-21 15:24:06.363118 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 9 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fd92402aef0 con 0x7fd930000e70
  534. 2014-05-21 15:24:06.363153 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 10 ==== osd_map(358049..358049 src has 350946..358049) v3 ==== 170826+0+0 (955659184 0 0) 0x7fd924054da0 con 0x7fd930000e70
  535. 2014-05-21 15:24:06.363165 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 11 ==== mon_subscribe_ack(300s) v1 ==== 20+0+0 (96432707 0 0) 0x7fd924055240 con 0x7fd930000e70
  536. 2014-05-21 15:24:06.378787 7fd942104700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- ?+0 0x7fd93c0172b0 con 0x7fd930000e70
  537. 2014-05-21 15:24:06.382451 7fd93b5fe700 1 -- 192.168.100.113:0/1017473 <== mon.2 192.168.100.114:6789/0 12 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+29582 (1092875540 0 1116028220) 0x7fd924001790 con 0x7fd930000e70
  538. 2014-05-21 15:24:06.525037 7fd942104700 1 -- 192.168.100.113:0/1017473 --> 192.168.100.114:6789/0 -- mon_command({"prefix": "osd crush create-or-move", "args": ["host=storage0113-ib", "root=default"], "id": 67, "weight": 2.73} v 0) v1 -- ?+0 0x7fd93c0172b0 con 0x7fd930000e70
  539. failed: 'timeout 30 /usr/bin/ceph -c /etc/ceph/ceph.conf --name=osd.67 --keyring=/var/lib/ceph/osd/ceph-67/keyring osd crush create-or-move -- 67 2.73 host=storage0113-ib root=default'
  540. [root@storage0113-ib ~]#
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!