[root@mon-01 ~]# rados bench -p volumes 60 write -t 64 --no-cleanup
 Maintaining 64 concurrent writes of 4194304 bytes for up to 60 seconds or 0 objects
 Object prefix: benchmark_data_mon-01.ceph.thegcloud.com_31157
   sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
     0       8         8         0         0         0         -         0
     1      53        53         0         0         0         -         0
     2      63        96        33   65.8509        66   1.26113   1.45424
     3      63       136        73   97.1825       160   1.34032   1.41162
     4      63       172       109    108.87       144    1.5127   1.51678
     5      63       210       147   117.485       152   1.49935   1.53706
     6      63       249       186    123.84       156   1.25703   1.54479
     7      63       291       228    130.14       168    1.3213   1.54243
     8      63       327       264   131.864       144   1.29792   1.52547
     9      63       368       305   135.429       164   1.34416    1.6059
    10      63       409       346   138.261       164   1.19749   1.58561
    11      63       446       383   139.143       148   1.34264   1.58565
    12      63       484       421   140.205       152   1.41115   1.60436
    13      63       519       456   140.171       140    1.4806   1.62289
    14      63       553       490   139.872       136   1.42325   1.61266
    15      63       592       529   140.945       156   1.36505   1.61804
    16      63       632       569   142.047       160   1.32348   1.59786
    17      63       670       607   142.579       152   1.28698   1.61432
    18      63       708       645     143.1       152   1.33547   1.62635
    19      63       748       685   143.986       160   1.35829   1.63638
2015-07-06 12:42:29.385800min lat: 1.14867 max lat: 5.64546 avg lat: 1.62814
   sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
    20      63       784       721   143.986       144   1.24867   1.62814
    21      63       822       759   144.366       152   1.36237   1.63641
    22      63       860       797   144.712       152   1.34844   1.62797
    23      63       898       835   145.028       152   5.42636   1.63091
    24      64       934       870   144.818       140   1.36549   1.63137
    25      63       967       904   144.465       136   1.42356   1.63955
    26      63      1006       943   144.907       156   1.33913     1.647
    27      63      1043       980    145.02       148   1.29812   1.65027
    28      63      1080      1017   145.125       148   1.25289   1.63744
    29      63      1122      1059   145.913       168   1.34306   1.64981
    30      63      1157      1094   145.708       140   1.38651    1.6559
    31      63      1197      1134   146.168       160   1.36965   1.65949
    32      63      1232      1169   145.974       140   1.47165    1.6556
    33      63      1274      1211   146.641       168   1.38816   1.66148
    34      63      1313      1250   146.914       156   1.34197   1.66105
    35      63      1345      1282   146.374       128   1.32719   1.65936
    36      63      1381      1318   146.307       144   1.39885   1.67112
    37      63      1425      1362   147.081       176   1.36102   1.66999
    38      63      1463      1400   147.181       152    1.2452   1.66211
    39      63      1509      1446   148.123       184    2.5361    1.6644
2015-07-06 12:42:49.404805min lat: 1.14867 max lat: 7.01721 avg lat: 1.66412
   sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
    40      63      1545      1482    148.02       144   1.34031   1.66412
    41      63      1585      1522   148.284       160   1.31721   1.66131
    42      63      1623      1560   148.372       152   1.36256   1.66685
    43      63      1662      1599   148.549       156   1.39212   1.66828
    44      63      1700      1637   148.627       152   1.38068   1.66794
    45      63      1739      1676   148.789       156    1.4131   1.66355
    46      63      1778      1715   148.945       156    1.4122   1.66877
    47      63      1815      1752   148.904       148   2.74821   1.66951
    48      63      1854      1791   149.052       156   1.25186   1.66708
    49      63      1892      1829   149.111       152   1.30747    1.6656
    50      63      1929      1866   149.088       148   1.32248   1.66797
    51      63      1965      1902   148.987       144   2.89986   1.66758
    52      63      2007      1944   149.352       168   1.36021   1.67063
    53      63      2045      1982   149.401       152   1.34285   1.66692
    54      63      2083      2020   149.449       152   2.71311   1.66837
    55      63      2123      2060    149.64       160   1.47042   1.67081
    56      63      2164      2101   149.895       164   1.38445   1.66979
    57      63      2197      2134   149.581       132   1.48055   1.66586
    58      63      2238      2175   149.829       164   2.87718   1.66844
    59      63      2276      2213   149.863       152   1.47092   1.66752
2015-07-06 12:43:09.423535min lat: 1.14867 max lat: 7.01721 avg lat: 1.66852
   sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
    60      64      2314      2250   149.832       148   2.87324   1.66852
 Total time run:         60.821720
Total writes made:      2314
Write size:             4194304
Bandwidth (MB/sec):     152.182

Stddev Bandwidth:       31.0878
Max bandwidth (MB/sec): 184
Min bandwidth (MB/sec): 0
Average Latency:        1.66929
Stddev Latency:         0.94213
Max latency:            7.01721
Min latency:            0.829686


[root@mon-01 ~]# rados bench -p volumes 60 seq
   sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
     0       0         0         0         0         0         -         0
     1      16       143       127   507.778       508 0.0252339  0.104903
     2      16       279       263   525.733       544 0.0380194   0.10957
     3      16       406       390   519.649       508 0.0352337   0.10969
     4      16       486       470    469.27       320  0.931778  0.125646
     5      16       604       588   469.767       472 0.0412659  0.130847
     6      16       718       702   467.467       456 0.0306929  0.130846
     7      16       829       813   464.109       444 0.0336499   0.13086
     8      16       940       924   461.445       444 0.0720624  0.132851
     9      16      1050      1034   459.058       440 0.0209541  0.135295
    10      16      1175      1159   463.142       500 0.0400018  0.135125
    11      16      1306      1290   468.663       524 0.0175081  0.134033
    12      16      1425      1409   469.264       476  0.807035  0.133344
    13      16      1558      1542   474.073       532 0.0224553  0.132409
    14      16      1649      1633   466.183       364 0.0278878  0.134562
    15      16      1772      1756   467.899       492 0.0320296  0.135038
    16      16      1894      1878   469.151       488 0.0159902   0.13439
    17      16      2006      1990   467.867       448 0.0218994  0.134705
    18      16      2121      2105   467.427       460 0.0318755  0.135103
    19      16      2247      2231   469.348       504 0.0279778  0.135102
 Total time run:        19.810823
Total reads made:     2314
Read size:            4194304
Bandwidth (MB/sec):    467.219

Average Latency:       0.136815
Max latency:           1.18576
Min latency:           0.0112911