# begin crush map
tunable choose_local_tries 0
tunable choose_local_fallback_tries 0
tunable choose_total_tries 50
tunable chooseleaf_descend_once 1
tunable chooseleaf_vary_r 1
tunable chooseleaf_stable 1
tunable straw_calc_version 1
tunable allowed_bucket_algs 54

# devices
device 0 osd.0 class hdd
device 1 osd.1 class hdd
device 2 osd.2 class hdd
device 3 osd.3 class ssd
device 4 osd.4 class hdd
device 5 osd.5 class hdd
device 6 osd.6 class hdd
device 7 osd.7 class hdd
device 8 osd.8 class nvme
device 9 osd.9 class hdd
device 10 osd.10 class ssd
device 11 osd.11 class ssd
device 12 osd.12 class ssd
device 13 osd.13 class nvme
device 14 osd.14 class ssd
device 15 osd.15 class ssd
device 16 osd.16 class nvme
device 17 osd.17 class hdd

# types
type 0 osd
type 1 host
type 2 chassis
type 3 rack
type 4 row
type 5 pdu
type 6 pod
type 7 room
type 8 datacenter
type 9 region
type 10 root

# buckets
host pve23 {
	id -13		# do not change unnecessarily
	id -14 class hdd		# do not change unnecessarily
	id -15 class ssd		# do not change unnecessarily
	id -10 class nvme		# do not change unnecessarily
	# weight 19.738
	alg straw2
	hash 0	# rjenkins1
	item osd.2 weight 2.729
	item osd.3 weight 0.109
	item osd.14 weight 0.109
	item osd.0 weight 3.639
	item osd.4 weight 1.364
	item osd.9 weight 10.914
	item osd.13 weight 0.873
}
host pve22 {
	id -6		# do not change unnecessarily
	id -7 class hdd		# do not change unnecessarily
	id -8 class ssd		# do not change unnecessarily
	id -11 class nvme		# do not change unnecessarily
	# weight 18.373
	alg straw2
	hash 0	# rjenkins1
	item osd.10 weight 0.109
	item osd.11 weight 0.109
	item osd.5 weight 2.729
	item osd.17 weight 3.639
	item osd.1 weight 10.914
	item osd.16 weight 0.873
}
host pve21 {
	id -2		# do not change unnecessarily
	id -3 class hdd		# do not change unnecessarily
	id -5 class ssd		# do not change unnecessarily
	id -12 class nvme		# do not change unnecessarily
	# weight 14.735
	alg straw2
	hash 0	# rjenkins1
	item osd.12 weight 0.109
	item osd.15 weight 0.109
	item osd.6 weight 2.729
	item osd.7 weight 10.914
	item osd.8 weight 0.873
}
root default {
	id -1		# do not change unnecessarily
	id -4 class hdd		# do not change unnecessarily
	id -9 class ssd		# do not change unnecessarily
	id -16 class nvme		# do not change unnecessarily
	# weight 52.846
	alg straw2
	hash 0	# rjenkins1
	item pve23 weight 19.738
	item pve22 weight 18.373
	item pve21 weight 14.735
}

# rules
rule replicated_rule {
	id 0
	type replicated
	min_size 1
	max_size 10
	step take default class hdd
	step chooseleaf firstn 0 type host
	step emit
}
rule ssd_ruleset {
	id 1
	type replicated
	min_size 1
	max_size 10
	step take default class ssd
	step chooseleaf firstn 0 type host
	step emit
}
rule nvme_rule {
	id 2
	type replicated
	min_size 1
	max_size 10
	step take default class nvme
	step chooseleaf firstn 0 type host
	step emit
}

# end crush map

[global]
	 auth_client_required = cephx
	 auth_cluster_required = cephx
	 auth_service_required = cephx
	 #bluestore_block_db_size = 13106127360
	 #bluestore_block_wal_size = 13106127360
	 cluster_network = 172.16.1.0/16
	 debug_asok = 0/0
	 debug_auth = 0/0
	 debug_buffer = 0/0
	 debug_client =  0/0
	 debug_context = 0/0
	 debug_crush = 0/0
	 debug_filer = 0/0
	 debug_filestore = 0/0
	 debug_finisher = 0/0
	 debug_heartbeatmap = 0/0
	 debug_journal = 0/0
	 debug_journaler = 0/0
	 debug_lockdep = 0/0
	 debug_mds =  0/0
	 debug_mds_balancer = 0/0
	 debug_mds_locker =  0/0
	 debug_mds_log =  0/0
	 debug_mds_log_expire =  0/0
	 debug_mds_migrator =  0/0
	 debug_mon = 0/0
	 debug_monc = 0/0
	 debug_ms = 0/0
	 debug_objclass = 0/0
	 debug_objectcacher = 0/0
	 debug_objecter = 0/0
	 debug_optracker = 0/0
	 debug_osd = 0/0
	 debug_paxos = 0/0
	 debug_perfcounter = 0/0
	 debug_rados = 0/0
	 debug_rbd = 0/0
	 debug_rgw = 0/0
	 debug_throttle = 0/0
	 debug_timer = 0/0
	 debug_tp = 0/0
	 fsid = e44fbe1c-b1c7-481d-bd25-dc595eae2d13
	 mon_allow_pool_delete = true
	 mon_host = 192.168.1.21 192.168.1.22 192.168.1.23
	 mon_max_pg_per_osd = 500
	 mon_osd_allow_primary_affinity = true
	 osd_journal_size = 28120
	 osd_max_backfills = 5
	 osd_max_pg_per_osd_hard_ratio = 3
	 osd_pool_default_min_size = 2
	 osd_pool_default_size = 3
	 osd_recovery_max_active = 6
	 osd_recovery_op_priority = 3
	 osd_scrub_auto_repair = true
	 osd_scrub_begin_hour = 1
	 osd_scrub_end_hour = 8
	 osd_scrub_sleep = 0.1
	 public_network = 192.168.1.0/24
	 rbd_cache = true
         bluestore_cache_size_hdd = 2147483648 # 2G
	bluestore_default_buffered_write = true # BlueStore has the ability to perform buffered writes. Buffered writes enable populating the read cache during the write process. This setting, in effect, changes the BlueStore cache into a write-through cache.
	# It is advised that spinning media continue to use 64 kB while SSD/NVMe are likely to benefit from setting to 4 kB.
	min_alloc_size_ssd=4096
	min_alloc_size_hdd=65536
	# https://yourcmc.ru/wiki/Ceph_performance
	bluefs_preextend_wal_files = true
        cephx_require_signatures = true
	cephx_cluster_require_signatures = true
	cephx_sign_messages = true

[client]
	 client_reconnect_stale = true
	 keyring = /etc/pve/priv/$cluster.$name.keyring

[mds]
	 keyring = /var/lib/ceph/mds/ceph-$id/keyring
	 mds_data = /var/lib/ceph/mds/ceph-$id

[mon]
	 mon_compact_on_start = true
	 mon_compact_on_trim = true

[osd]
	 filestore_xattr_use_omap = true
	 keyring = /var/lib/ceph/osd/ceph-$id/keyring
	 osd_crush_update_on_start = true

[mds.pve23]
	 host = 192.168.1.23

[mds.pve21]
	 host = 192.168.1.21

[mds.pve22]
	 host = 192.168.1.22

root@pve21:~# rados bench -p zfs_cache 30 write --no-cleanup
hints = 1
Maintaining 16 concurrent writes of 4194304 bytes to objects of size 4194304 for up to 30 seconds or 0 objects
Object prefix: benchmark_data_pve21_3994238
  sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
    0       0         0         0         0         0           -           0
    1      16        51        35   139.992       140    0.209004    0.282963
    2      16        87        71   141.989       144    0.339198     0.37351
    3      16       124       108    143.99       148    0.854306    0.395849
    4      16       157       141   140.989       132    0.877978    0.409175
    5      16       193       177   141.589       144   0.0135585    0.421509
    6      16       224       208   138.654       124    0.689556    0.437592
    7      16       262       246   140.559       152    0.604785    0.435107
    8      16       295       279   139.487       132   0.0949523    0.438708
    9      16       332       316   140.431       148    0.842605    0.439193
   10      16       366       350   139.986       136   0.0140917    0.442065
   11      16       397       381   138.532       124    0.679741    0.449535
   12      16       431       415   138.319       136    0.674606    0.451588
   13      16       473       457   140.601       168   0.0133488     0.44542
   14      16       511       495   141.414       152    0.292574    0.441756
   15      16       545       529   141.052       136    0.923989    0.441713
   16      16       593       577   144.235       192    0.012868    0.435535
   17      16       631       615   144.691       152   0.0129975     0.43247
   18      16       670       654   145.319       156   0.0145094    0.431384
   19      16       705       689   145.038       140    0.818859    0.432818
2020-11-05 20:42:52.172037 min lat: 0.0126369 max lat: 0.999587 avg lat: 0.434105
  sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
   20      16       743       727   145.385       152     0.60138    0.434105
   21      16       778       762   145.128       140    0.316159    0.433479
   22      16       818       802   145.803       160    0.334659    0.431086
   23      16       849       833   144.854       124    0.902827    0.434864
   24      16       881       865   144.152       128    0.362459    0.437654
   25      16       912       896   143.345       124    0.776994    0.440992
   26      16       955       939   144.447       172    0.289116    0.437467
   27      16       994       978   144.874       156    0.774429    0.436769
   28      16      1035      1019   145.556       164    0.528945    0.435309
   29      16      1073      1057   145.778       152    0.796811    0.433397
   30      16      1114      1098   146.385       164    0.733767    0.433262
Total time run:         30.6479
Total writes made:      1115
Write size:             4194304
Object size:            4194304
Bandwidth (MB/sec):     145.524
Stddev Bandwidth:       16.1578
Max bandwidth (MB/sec): 192
Min bandwidth (MB/sec): 124
Average IOPS:           36
Stddev IOPS:            4.03946
Max IOPS:               48
Min IOPS:               31
Average Latency(s):     0.43931
Stddev Latency(s):      0.316067
Max latency(s):         1.15206
Min latency(s):         0.0124192

root@pve21:~# rados bench -p zfs_cache 30 seq
hints = 1
  sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
    0       0         0         0         0         0           -           0
    1      16        51        35   139.976       140    0.152329    0.261514
    2      16        89        73    145.98       152    0.967852    0.346284
    3      16       125       109   145.316       144    0.929344    0.383067
    4      16       160       144   143.983       140    0.142341    0.395781
    5      16       194       178   142.384       136    0.826413    0.408423
    6      16       226       210   139.984       128    0.142744    0.424383
    7      16       265       249    142.27       156  0.00575118    0.426178
    8      16       298       282   140.985       132    0.720539    0.435783
    9      16       334       318   141.318       144    0.872076    0.433645
   10      16       368       352   140.785       136    0.872145    0.435896
   11      16       399       383   139.258       124    0.142806    0.439325
   12      16       434       418   139.319       140    0.781405    0.445867
   13      16       477       461   141.832       172    0.889309    0.437065
   14      16       513       497   141.986       144      0.9639    0.436264
   15      16       548       532   141.853       140    0.229899    0.437694
   16      16       597       581   145.235       196    0.217501    0.430265
   17      16       638       622   146.338       164  0.00815341    0.424903
   18      16       674       658   146.207       144    0.787708    0.427804
   19      16       711       695     146.3       148  0.00604705    0.426704
2020-11-05 20:49:50.333587 min lat: 0.00335944 max lat: 1.09719 avg lat: 0.426499
  sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
   20      16       751       735   146.985       160    0.139573    0.426499
   21      16       787       771   146.841       144  0.00424377    0.428744
   22      16       824       808   146.893       148  0.00610794    0.428274
   23      16       854       838   145.724       120   0.0777444    0.430503
   24      16       887       871   145.151       132   0.0738695    0.432835
   25      16       917       901   144.145       120   0.0736379    0.436277
   26      16       962       946   145.523       180    0.142901     0.43178
   27      16      1006       990   146.651       176    0.141083    0.428757
   28      16      1046      1030   147.127       160    0.727332    0.428981
   29      16      1083      1067   147.157       148   0.0791994    0.427605
   30      15      1115      1100   146.652       132    0.226387    0.427832
Total time run:       30.3715
Total reads made:     1115
Read size:            4194304
Object size:          4194304
Bandwidth (MB/sec):   146.848
Average IOPS:         36
Stddev IOPS:          4.45153
Max IOPS:             49
Min IOPS:             30
Average Latency(s):   0.434557
Max latency(s):       1.49625
Min latency(s):       0.00334497