performance

rados bench results:

Concurrency 	Replication size	       Write(MB/s)     Seq Read(MB/s)
32		1 			13.5		32.8
32		2			12.7		32.0
32		3			6.1		30.2

Commands I used (Pool size was updated appropriately):

rados -p performance_test bench 120 write -b 8192 -t 100 --no-cleanup
rados -p performance_test bench 120 seq -t 100



1) Disk tests - All have similar numbers:
# dd if=/dev/zero of=here bs=1G count=1 oflag=direct
1+0 records in
1+0 records out
1073741824 bytes (1.1 GB) copied, 10.0691 s, 107 MB/s

2) 10G network is not holding up
# iperf -c 10.13.10.15  -i2 -t 10

------------------------------------------------------------

Client connecting to 10.13.10.15, TCP port 5001

TCP window size: 85.0 KByte (default)

------------------------------------------------------------

[  3] local 10.13.30.13 port 56459 connected with 10.13.10.15 port 5001

[ ID] Interval       Transfer     Bandwidth

[  3]  0.0- 2.0 sec  2.17 GBytes  9.33 Gbits/sec

[  3]  2.0- 4.0 sec  2.18 GBytes  9.37 Gbits/sec

[  3]  4.0- 6.0 sec  2.18 GBytes  9.37 Gbits/sec

[  3]  6.0- 8.0 sec  2.18 GBytes  9.38 Gbits/sec

[  3]  8.0-10.0 sec  2.18 GBytes  9.37 Gbits/sec

[  3]  0.0-10.0 sec  10.9 GBytes  9.36 Gbits/sec



3) Ceph Status

# ceph health
HEALTH_OK
root@us1-r04u05s01-ceph:~# ceph status
    cluster 5e95b6fa-0b99-4c31-8aa9-7a88h6hc5eda
     health HEALTH_OK
     monmap e4: 4 mons at {us1-r01u05s01-ceph=10.1.30.10:6789/0,us1-r01u09s01-ceph=10.1.30.11:6789/0,us1-r04u05s01-ceph=10.1.30.14:6789/0,us1-r04u09s01-ceph=10.1.30.15:6789/0}, election epoch 78, quorum 0,1,2,3 us1-r01u05s01-ceph,us1-r01u09s01-ceph,us1-r04u05s01-ceph,us1-r04u09s01-ceph
     osdmap e1029: 97 osds: 97 up, 97 in
      pgmap v1850869: 12480 pgs, 6 pools, 587 GB data, 116 kobjects
            1787 GB used, 318 TB / 320 TB avail
               12480 active+clean
  client io 0 B/s rd, 25460 B/s wr, 20 op/s


4) Ceph configuration
# cat ceph.conf

[global]
  auth cluster required = cephx
  auth service required = cephx
  auth client required = cephx
  cephx require signatures = True
  cephx cluster require signatures = True
  cephx service require signatures = False
  fsid = 5e95b6fa-0b99-4c31-8aa9-7a88h6hc5eda
  osd pool default pg num = 4096
  osd pool default pgp num = 4096
  osd pool default size = 3
  osd pool default min size = 1
  osd pool default crush rule = 0
  # Disable in-memory logs
  debug_lockdep = 0/0
  debug_context = 0/0
  debug_crush = 0/0
  debug_buffer = 0/0
  debug_timer = 0/0
  debug_filer = 0/0
  debug_objecter = 0/0
  debug_rados = 0/0
  debug_rbd = 0/0
  debug_journaler = 0/0
  debug_objectcatcher = 0/0
  debug_client = 0/0
  debug_osd = 0/0
  debug_optracker = 0/0
  debug_objclass = 0/0
  debug_filestore = 0/0
  debug_journal = 0/0
  debug_ms = 0/0
  debug_monc = 0/0
  debug_tp = 0/0
  debug_auth = 0/0
  debug_finisher = 0/0
  debug_heartbeatmap = 0/0
  debug_perfcounter = 0/0
  debug_asok = 0/0
  debug_throttle = 0/0
  debug_mon = 0/0
  debug_paxos = 0/0
  debug_rgw = 0/0

[mon]
  mon osd down out interval = 600
  mon osd min down reporters = 2
    [mon.us1-r01u05s01-ceph]
    host = us1-r01u05s01-ceph
    mon addr = 10.1.30.10
      [mon.us1-r01u09s01-ceph]
    host = us1-r01u09s01-ceph
    mon addr = 10.1.30.11
      [mon.us1-r04u05s01-ceph]
    host = us1-r04u05s01-ceph
    mon addr = 10.1.30.14
      [mon.us1-r04u09s01-ceph]
    host = us1-r04u09s01-ceph
    mon addr = 10.1.30.15

[osd]
  osd mkfs type = xfs
  osd mkfs options xfs = -f -i size=2048
  osd mount options xfs = noatime
  osd journal size = 10000
  cluster_network = 10.2.0.0/16
  public_network = 10.1.0.0/16
  osd mon heartbeat interval = 30
  # Performance tuning
  filestore merge threshold = 40
  filestore split multiple = 8
  osd op threads = 8
  filestore op threads = 8
  filestore max sync interval = 5
  osd max scrubs = 1
  # Recovery tuning
  osd recovery max active = 5
  osd max backfills = 2
  osd recovery op priority = 2
  osd recovery max chunk = 8388608
  osd recovery threads = 1
  osd objectstore = filestore
  osd crush update on start = true

[mds]