ceph@node-1:/# rados bench -p ceph 10 write  --no-cleanup
Maintaining 16 concurrent writes of 4194304 bytes for up to 10 seconds or 0 objects
Object prefix: benchmark_data_node-1_13109
  sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
    0       0         0         0         0         0           -           0
    1      16       114        98    391.85       392   0.0470227    0.137642
    2      16       165       149   297.919       204    0.106653    0.188288
    3      16       198       182   242.611       132    0.475566    0.231503
    4      16       238       222   221.954       160    0.676444    0.274919
    5      16       279       263    210.36       164    0.497056    0.284888
    6      16       333       317   211.294       216    0.262457    0.292621
    7      16       375       359   205.105       168    0.377141    0.299099
    8      16       424       408   203.964       196   0.0727931    0.303847
    9      16       471       455   202.187       188    0.449751    0.307538
   10      16       521       505   201.966       200    0.249628    0.312503
Total time run:         10.287073
Total writes made:      522
Write size:             4194304
Bandwidth (MB/sec):     202.973
Stddev Bandwidth:       91.0464
Max bandwidth (MB/sec): 392
Min bandwidth (MB/sec): 0
Average IOPS:           50
Average Latency(s):     0.315181
Stddev Latency(s):      0.186956
Max latency(s):         0.91821
Min latency(s):         0.0305284
ceph@node-1:/# rados bench -p ceph 10 seq
  sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
    0       0         0         0         0         0           -           0
    1      15       449       434   1733.99      1736    0.026892   0.0342909
Total time run:       1.229247
Total reads made:     522
Read size:            4194304
Bandwidth (MB/sec):   1698.6
Average IOPS:         424
Average Latency(s):   0.0358385
Max latency(s):       0.152383
Min latency(s):       0.00553852

ceph@node-1:/# rados bench -p ceph 10 rand
  sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
    0       0         0         0         0         0           -           0
    1      15       410       395   1579.73      1580   0.0249911   0.0363033
    2      15       851       836    1671.7      1764   0.0202046   0.0362797
    3      16      1318      1302   1735.71      1864   0.0238266   0.0351212
    4      15      1771      1756   1755.73      1816  0.00782606   0.0348113
    5      15      2228      2213   1770.13      1828    0.030824   0.0345134
    6      16      2656      2640   1759.74      1708  0.00659975    0.034742
    7      16      3100      3084   1762.03      1776  0.00655059   0.0347319
    8      15      3548      3533   1766.24      1796   0.0253535   0.0347375
    9      16      3976      3960   1759.74      1708   0.0342474   0.0348734
   10      15      4405      4390   1755.75      1720    0.011223   0.0349438
Total time run:       10.053505
Total reads made:     4406
Read size:            4194304
Bandwidth (MB/sec):   1753.02
Average IOPS:         438
Average Latency(s):   0.0350626
Max latency(s):       0.361596
Min latency(s):       0.0051373


ceph@node-1:/# dd if=/dev/zero of=/var/lib/ceph/osd/ceph-0/tmpfile bs=300M count=1 oflag=direct
1+0 records in
1+0 records out
314572800 bytes (315 MB) copied, 1.2567 s, 250 MB/s
ceph@node-1:/# rm /var/lib/ceph/osd/ceph-0/tmpfile
ceph@node-1:/# dd if=/dev/zero of=/var/lib/ceph/osd/ceph-1/tmpfile bs=300M count=1 oflag=direct
1+0 records in
1+0 records out
314572800 bytes (315 MB) copied, 1.18015 s, 267 MB/s
ceph@node-1:/# rm /var/lib/ceph/osd/ceph-1/tmpfile
ceph@node-1:/# dd if=/dev/zero of=/var/lib/ceph/osd/ceph-2/tmpfile bs=300M count=1 oflag=direct
1+0 records in
1+0 records out
314572800 bytes (315 MB) copied, 1.14226 s, 275 MB/s
ceph@node-1:/# rm /var/lib/ceph/osd/ceph-2/tmpfile
ceph@node-1:/# dd if=/dev/zero of=/var/lib/ceph/osd/ceph-3/tmpfile bs=300M count=1 oflag=direct
1+0 records in
1+0 records out
314572800 bytes (315 MB) copied, 1.3018 s, 242 MB/s
ceph@node-1:/# rm /var/lib/ceph/osd/ceph-3/tmpfile
ceph@node-1:/# dd if=/dev/zero of=/var/lib/ceph/osd/ceph-4/tmpfile bs=300M count=1 oflag=direct
1+0 records in
1+0 records out
314572800 bytes (315 MB) copied, 1.24705 s, 252 MB/s
ceph@node-1:/# rm /var/lib/ceph/osd/ceph-4/tmpfile
ceph@node-1:/# dd if=/dev/zero of=/var/lib/ceph/osd/ceph-5/tmpfile bs=300M count=1 oflag=direct
1+0 records in
1+0 records out
314572800 bytes (315 MB) copied, 1.1693 s, 269 MB/s
ceph@node-1:/# rm /var/lib/ceph/osd/ceph-5/tmpfile