#####################
# 50GB #1 (1m44s)
#####################
~# time fio --name=writefile --size=50G --filesize=50G --filename=/dev/rbd0 --bs=1M --nrfiles=1 --direct=1 --sync=0 --randrepeat=0 --rw=write --refill_buffers --end_fsync=1 --iodepth=200 --ioengine=libaio
writefile: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=libaio, iodepth=200
fio 1.59
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/485.4M /s] [0 /474  iops] [eta 00m:00s]
writefile: (groupid=0, jobs=1): err= 0: pid=45723
  write: io=51200MB, bw=502962KB/s, iops=491 , runt=104240msec
    slat (usec): min=24 , max=584611 , avg=1806.77, stdev=9912.97
    clat (msec): min=139 , max=1296 , avg=404.55, stdev=121.13
     lat (msec): min=139 , max=1296 , avg=406.36, stdev=121.35
    bw (KB/s) : min=    0, max=716800, per=50.17%, avg=252328.32, stdev=265995.18
  cpu          : usr=9.04%, sys=3.74%, ctx=7356, majf=0, minf=2715887
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
     issued r/w/d: total=0/51200/0, short=0/0/0

     lat (msec): 250=1.91%, 500=80.36%, 750=15.86%, 1000=1.68%, 2000=0.19%

Run status group 0 (all jobs):
  WRITE: io=51200MB, aggrb=502962KB/s, minb=515033KB/s, maxb=515033KB/s, mint=104240msec, maxt=104240msec

Disk stats (read/write):
  rbd0: ios=136/102236, merge=0/0, ticks=48/14720048, in_queue=14729576, util=99.90%

real	1m44.649s
user	0m9.576s
sys	0m4.368s

#####################
# 50GB #2 (1m49s)
#####################
~# time fio --name=writefile --size=50G --filesize=50G --filename=/dev/rbd0 --bs=1M --nrfiles=1 --direct=1 --sync=0 --randrepeat=0 --rw=write --refill_buffers --end_fsync=1 --iodepth=200 --ioengine=libaio
writefile: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=libaio, iodepth=200
fio 1.59
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/474.7M /s] [0 /463  iops] [eta 00m:00s]
writefile: (groupid=0, jobs=1): err= 0: pid=45801
  write: io=51200MB, bw=479301KB/s, iops=468 , runt=109386msec
    slat (usec): min=23 , max=502763 , avg=1906.38, stdev=10684.49
    clat (msec): min=107 , max=2101 , avg=424.29, stdev=159.67
     lat (msec): min=107 , max=2121 , avg=426.20, stdev=160.14
    bw (KB/s) : min=49420, max=718848, per=100.77%, avg=482967.42, stdev=140410.27
  cpu          : usr=8.56%, sys=3.67%, ctx=7904, majf=0, minf=3206745
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
     issued r/w/d: total=0/51200/0, short=0/0/0

     lat (msec): 250=1.40%, 500=77.37%, 750=17.06%, 1000=3.43%, 2000=0.73%
     lat (msec): >=2000=0.01%

Run status group 0 (all jobs):
  WRITE: io=51200MB, aggrb=479300KB/s, minb=490804KB/s, maxb=490804KB/s, mint=109386msec, maxt=109386msec

Disk stats (read/write):
  rbd0: ios=131/102172, merge=0/0, ticks=52/15424180, in_queue=15449352, util=99.92%

real	1m49.801s
user	0m9.464s
sys	0m4.400s

#####################
# 50GB #3 (1m40s)
#####################
~# time fio --name=writefile --size=50G --filesize=50G --filename=/dev/rbd0 --bs=1M --nrfiles=1 --direct=1 --sync=0 --randrepeat=0 --rw=write --refill_buffers --end_fsync=1 --iodepth=200 --ioengine=libaio
writefile: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=libaio, iodepth=200
fio 1.59
Starting 1 process
Jobs: 1 (f=1): [W] [100.0% done] [0K/572.9M /s] [0 /559  iops] [eta 00m:00s]
writefile: (groupid=0, jobs=1): err= 0: pid=46447
  write: io=51200MB, bw=524267KB/s, iops=511 , runt=100004msec
    slat (usec): min=24 , max=318755 , avg=1700.94, stdev=9027.42
    clat (msec): min=151 , max=924 , avg=388.13, stdev=117.44
     lat (msec): min=151 , max=924 , avg=389.83, stdev=117.61
    bw (KB/s) : min=118082, max=749858, per=100.40%, avg=526386.36, stdev=132177.94
  cpu          : usr=9.61%, sys=4.85%, ctx=9961, majf=0, minf=4099449
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
     issued r/w/d: total=0/51200/0, short=0/0/0

     lat (msec): 250=3.38%, 500=80.03%, 750=15.52%, 1000=1.06%

Run status group 0 (all jobs):
  WRITE: io=51200MB, aggrb=524267KB/s, minb=536849KB/s, maxb=536849KB/s, mint=100004msec, maxt=100004msec

Disk stats (read/write):
  rbd0: ios=128/102227, merge=0/0, ticks=48/14092984, in_queue=14110840, util=99.90%

real	1m40.381s
user	0m9.700s
sys	0m5.216s

#####################
# 75GB #1 (10m12s)
#####################
root@node15.privatestack:~# time fio --name=writefile --size=75G --filesize=75G --filename=/dev/rbd0 --bs=1M --nrfiles=1 --direct=1 --sync=0 --randrepeat=0 --rw=write --refill_buffers --end_fsync=1 --iodepth=200 --ioengine=libaio
writefile: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=libaio, iodepth=200
fio 1.59
Starting 1 process
Jobs: 1 (f=1): [W] [99.8% done] [0K/0K /s] [0 /0  iops] [eta 00m:01s]
writefile: (groupid=0, jobs=1): err= 0: pid=46484
  write: io=76800MB, bw=128383KB/s, iops=125 , runt=612568msec
    slat (usec): min=23 , max=473875 , avg=1880.78, stdev=10286.89
    clat (msec): min=200 , max=463033 , avg=1586.88, stdev=22811.21
     lat (msec): min=200 , max=463033 , avg=1588.76, stdev=22811.19
    bw (KB/s) : min=    0, max=727952, per=229.69%, avg=294876.89, stdev=256836.05
  cpu          : usr=2.31%, sys=1.28%, ctx=14920, majf=0, minf=6698651
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
     issued r/w/d: total=0/76800/0, short=0/0/0

     lat (msec): 250=1.80%, 500=75.62%, 750=18.87%, 1000=2.99%, 2000=0.46%
     lat (msec): >=2000=0.26%

Run status group 0 (all jobs):
  WRITE: io=76800MB, aggrb=128382KB/s, minb=131463KB/s, maxb=131463KB/s, mint=612568msec, maxt=612568msec

Disk stats (read/write):
  rbd0: ios=127/153424, merge=0/0, ticks=52/23016768, in_queue=23140740, util=99.97%

real	10m12.957s
user	0m14.888s
sys	0m9.732s

#####################
# 75GB #2 (10m11s)
#####################
~# time fio --name=writefile --size=75G --filesize=75G --filename=/dev/rbd0 --bs=1M --nrfiles=1 --direct=1 --sync=0 --randrepeat=0 --rw=write --refill_buffers --end_fsync=1 --iodepth=200 --ioengine=libaio
writefile: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=libaio, iodepth=200
fio 1.59
Starting 1 process
Jobs: 1 (f=1): [W] [99.8% done] [0K/0K /s] [0 /0  iops] [eta 00m:01s]
writefile: (groupid=0, jobs=1): err= 0: pid=46566
  write: io=76800MB, bw=128687KB/s, iops=125 , runt=611118msec
    slat (usec): min=24 , max=378660 , avg=1797.48, stdev=9738.18
    clat (msec): min=191 , max=478627 , avg=1583.26, stdev=23088.92
     lat (msec): min=191 , max=478627 , avg=1585.05, stdev=23088.98
    bw (KB/s) : min=    0, max=727040, per=233.62%, avg=300637.36, stdev=258856.54
  cpu          : usr=2.36%, sys=1.30%, ctx=16315, majf=0, minf=6920927
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
     issued r/w/d: total=0/76800/0, short=0/0/0

     lat (msec): 250=1.50%, 500=80.91%, 750=16.09%, 1000=1.21%, 2000=0.03%
     lat (msec): >=2000=0.26%

Run status group 0 (all jobs):
  WRITE: io=76800MB, aggrb=128687KB/s, minb=131775KB/s, maxb=131775KB/s, mint=611118msec, maxt=611118msec

Disk stats (read/write):
  rbd0: ios=128/153443, merge=0/0, ticks=56/21877872, in_queue=22293696, util=99.98%

real	10m11.514s
user	0m14.956s
sys	0m9.820s

#####################
# 75GB #3 (10m13s)
#####################
~# time fio --name=writefile --size=75G --filesize=75G --filename=/dev/rbd0 --bs=1M --nrfiles=1 --direct=1 --sync=0 --randrepeat=0 --rw=write --refill_buffers --end_fsync=1 --iodepth=200 --ioengine=libaio
writefile: (g=0): rw=write, bs=1M-1M/1M-1M, ioengine=libaio, iodepth=200
fio 1.59
Starting 1 process
Jobs: 1 (f=1): [W] [99.8% done] [0K/0K /s] [0 /0  iops] [eta 00m:01s]
writefile: (groupid=0, jobs=1): err= 0: pid=47255
  write: io=76800MB, bw=128312KB/s, iops=125 , runt=612905msec
    slat (usec): min=24 , max=533254 , avg=1865.12, stdev=10362.34
    clat (msec): min=204 , max=464887 , avg=1587.69, stdev=22952.08
     lat (msec): min=204 , max=464887 , avg=1589.55, stdev=22952.09
    bw (KB/s) : min=    0, max=718848, per=231.36%, avg=296862.60, stdev=257964.55
  cpu          : usr=2.33%, sys=1.05%, ctx=16450, majf=0, minf=5267422
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=99.9%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.1%
     issued r/w/d: total=0/76800/0, short=0/0/0

     lat (msec): 250=1.74%, 500=79.06%, 750=16.12%, 1000=2.27%, 2000=0.55%
     lat (msec): >=2000=0.27%

Run status group 0 (all jobs):
  WRITE: io=76800MB, aggrb=128312KB/s, minb=131391KB/s, maxb=131391KB/s, mint=612905msec, maxt=612905msec

Disk stats (read/write):
  rbd0: ios=137/153268, merge=0/0, ticks=56/22676532, in_queue=22799260, util=99.95%

real	10m13.313s
user	0m14.808s
sys	0m8.284s