root@ubin:~# fio --name TEST --eta-newline=5s --filename=fio-tempfile.dat --rw=randwrite --size=2048m --io_size=10g --blocksize=4k --ioengine=libaio --fsync=1 --iodepth=32 --direct=1 --numjobs=1 --runtime=60 --group_reporting
TEST: (g=0): rw=randwrite, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=32
fio-3.1
Starting 1 process
TEST: Laying out IO file (1 file / 2048MiB)
Jobs: 1 (f=1): [w(1)][11.7%][r=0KiB/s,w=3799KiB/s][r=0,w=949 IOPS][eta 00m:53s]
Jobs: 1 (f=1): [w(1)][21.7%][r=0KiB/s,w=5861KiB/s][r=0,w=1465 IOPS][eta 00m:47s]
Jobs: 1 (f=1): [w(1)][31.7%][r=0KiB/s,w=4756KiB/s][r=0,w=1189 IOPS][eta 00m:41s]
Jobs: 1 (f=1): [w(1)][41.7%][r=0KiB/s,w=3815KiB/s][r=0,w=953 IOPS][eta 00m:35s]
Jobs: 1 (f=1): [w(1)][51.7%][r=0KiB/s,w=4216KiB/s][r=0,w=1054 IOPS][eta 00m:29s]
Jobs: 1 (f=1): [w(1)][61.7%][r=0KiB/s,w=3852KiB/s][r=0,w=963 IOPS][eta 00m:23s]
Jobs: 1 (f=1): [w(1)][71.7%][r=0KiB/s,w=4776KiB/s][r=0,w=1194 IOPS][eta 00m:17s]
Jobs: 1 (f=1): [w(1)][81.7%][r=0KiB/s,w=4468KiB/s][r=0,w=1117 IOPS][eta 00m:11s]
Jobs: 1 (f=1): [w(1)][91.7%][r=0KiB/s,w=4940KiB/s][r=0,w=1235 IOPS][eta 00m:05s]
Jobs: 1 (f=1): [w(1)][100.0%][r=0KiB/s,w=3844KiB/s][r=0,w=961 IOPS][eta 00m:00s]
TEST: (groupid=0, jobs=1): err= 0: pid=2754: Tue Sep 17 03:29:05 2019
  write: IOPS=1044, BW=4176KiB/s (4277kB/s)(245MiB/60001msec)
    slat (usec): min=5, max=2520, avg=33.09, stdev=49.08
    clat (usec): min=1058, max=152269, avg=29690.33, stdev=7094.45
     lat (usec): min=1103, max=152296, avg=29724.80, stdev=7100.39
    clat percentiles (msec):
     |  1.00th=[   19],  5.00th=[   21], 10.00th=[   23], 20.00th=[   25],
     | 30.00th=[   27], 40.00th=[   28], 50.00th=[   30], 60.00th=[   31],
     | 70.00th=[   32], 80.00th=[   34], 90.00th=[   36], 95.00th=[   40],
     | 99.00th=[   53], 99.50th=[   55], 99.90th=[   81], 99.95th=[  144],
     | 99.99th=[  150]
   bw (  KiB/s): min= 2136, max= 6064, per=99.97%, avg=4174.58, stdev=739.77, samples=120
   iops        : min=  534, max= 1516, avg=1043.63, stdev=184.95, samples=120
  lat (msec)   : 2=0.01%, 4=0.01%, 10=0.01%, 20=2.91%, 50=95.37%
  lat (msec)   : 100=1.61%, 250=0.10%
  cpu          : usr=1.96%, sys=5.26%, ctx=63778, majf=0, minf=9
  IO depths    : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=100.0%, >=64=0.0%
     submit    : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
     complete  : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.1%, 64=0.0%, >=64=0.0%
     issued rwt: total=0,62648,0, short=0,0,0, dropped=0,0,0
     latency   : target=0, window=0, percentile=100.00%, depth=32

Run status group 0 (all jobs):
  WRITE: bw=4176KiB/s (4277kB/s), 4176KiB/s-4176KiB/s (4277kB/s-4277kB/s), io=245MiB (257MB), run=60001-60001msec

Disk stats (read/write):
  sda: ios=0/190416, merge=0/190212, ticks=0/59156, in_queue=8296, util=12.07%