drbd9-write-speed-slow

[root@ae-srv06 ~]# zfs list
NAME               USED  AVAIL  REFER  MOUNTPOINT
drbdpool           495G  12.1T    19K  /drbdpool
drbdpool/aefs_00   495G  12.6T  22.5G  -
export             384K   988G   108K  /export
[root@ae-srv06 ~]# zfs create drbdpool/test
[root@ae-srv06 ~]# mkdir /drbdpool/test/minime
[root@ae-srv06 ~]# chown minime /drbdpool/test/minime/
[root@ae-srv06 ~]# su - minime
Last login: Wed May 17 11:21:42 EDT 2017 on pts/0
[minime@ae-srv06 ~]$ cd /drbdpool/test/minime/
[minime@ae-srv06 minime]$ dd if=/dev/zero of=/drbdpool/test/minime/testfile bs=1M count=100 conv=fsync && dd if=/drbdpool/test/minime/testfile of=/dev/null && /bin/rm /drbdpool/test/minime/testfile
100+0 records in
100+0 records out
104857600 bytes (105 MB) copied, 0.221458 s, 473 MB/s
204800+0 records in
204800+0 records out
104857600 bytes (105 MB) copied, 0.347231 s, 302 MB/s
[minime@ae-srv06 minime]$ bonnie++ -d /drbdpool/test/minime/ -r 16384
Writing a byte at a time...done
Writing intelligently...done
Rewriting...done
Reading a byte at a time...done
Reading intelligently...done
start 'em...done...done...done...done...done...
Create files in sequential order...done.
Stat files in sequential order...done.
Delete files in sequential order...done.
Create files in random order...done.
Stat files in random order...done.
Delete files in random order...done.
Version  1.97       ------Sequential Output------ --Sequential Input- --Random-
Concurrency   1     -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine        Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP  /sec %CP
ae-srv06        32G   203  99 695183  87 591090  87   605 100 2595180  97  7682 133
Latency             43249us    1953us     483us   14371us     937us     113us
Version  1.97       ------Sequential Create------ --------Random Create--------
ae-srv06            -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
              files  /sec %CP  /sec %CP  /sec %CP  /sec %CP  /sec %CP  /sec %CP
                 16  7505  99 +++++ +++  6191  99  5838  98 +++++ +++  8695  99
Latency             49445us     285us    1156us     105ms      20us     767us
1.97,1.97,ae-srv06,1,1495112276,32G,,203,99,695183,87,591090,87,605,100,2595180,97,7682,133,16,,,,,7505,99,+++++,+++,6191,99,5838,98,+++++,+++,8695,99,43249us,1953us,483us,14371us,937us,113us,49445us,285us,1156us,105ms,20us,767us

##############################
#
# Now test on drbd device
#
##############################

[root@ae-srv06 ~]# drbdadm status
.drbdctrl role:Primary
  volume:0 disk:UpToDate
  volume:1 disk:UpToDate
  ae-srv07 role:Secondary
    volume:0 peer-disk:UpToDate
    volume:1 peer-disk:UpToDate

aefs role:Primary
  disk:UpToDate
  ae-srv07 role:Secondary
    peer-disk:UpToDate

[root@ae-srv06 ~]# zfs mount export
[root@ae-srv06 ~]# df -h | grep export
export                 989G     0  989G   0% /export
[root@ae-srv06 ~]# su - minime
Last login: Thu May 18 08:14:32 EDT 2017 on pts/0
[minime@ae-srv06 ~]$ cd /export/minime/
[minime@ae-srv06 minime]$ dd if=/dev/zero of=/export/minime/testfile bs=1M count=100 conv=fsync && dd if=/export/minime/testfile of=/dev/null && /bin/rm /export/minime/testfile
100+0 records in
100+0 records out
104857600 bytes (105 MB) copied, 4.08932 s, 25.6 MB/s
204800+0 records in
204800+0 records out
104857600 bytes (105 MB) copied, 0.307471 s, 341 MB/s
[minime@ae-srv06 minime]$ bonnie++ -d /export/minime/ -r 16384                                                                                                                            Writing a byte at a time...done
Writing intelligently...done
Rewriting...done
Reading a byte at a time...done
Reading intelligently...done
start 'em...done...done...done...done...done...
Create files in sequential order...done.
Stat files in sequential order...done.
Delete files in sequential order...done.
Create files in random order...done.
Stat files in random order...done.
Delete files in random order...done.
Version  1.97       ------Sequential Output------ --Sequential Input- --Random-
Concurrency   1     -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
Machine        Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP  /sec %CP
ae-srv06        32G   208  99 21237   4 16181   4   560  97 2073494  94  2056 120
Latency             42442us    2106us    6504us   25420us   15741us   30045us
Version  1.97       ------Sequential Create------ --------Random Create--------
ae-srv06            -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
              files  /sec %CP  /sec %CP  /sec %CP  /sec %CP  /sec %CP  /sec %CP
                 16  6903  88 +++++ +++  5382  96  5408  92 +++++ +++  5173  95
Latency             49560us     293us    1980us     102ms      24us    1271us
1.97,1.97,ae-srv06,1,1495112104,32G,,208,99,21237,4,16181,4,560,97,2073494,94,2056,120,16,,,,,6903,88,+++++,+++,5382,96,5408,92,+++++,+++,5173,95,42442us,2106us,6504us,25420us,15741us,30045us,49560us,293us,1980us,102ms,24us,1271us