API tools faq
paste
View difference between Paste ID: 5WyqHuXd and h8iuzUQx
SHOW: | | - or go back to the newest paste.
1
rados bench results:
2
3-
Concurrency 	Replication size	       Write(MB/s)     Seq Read(MB/s)

3+
# rados -p performance_test bench 120 write -b 4194304 -t 100 --no-cleanup

4-
32		1 			13.5		32.8

4+
5-
32		2			12.7		32.0

5+
 Total time run:         120.133251

6-
32		3			6.1		30.2

6+
Total writes made:      17529

7
Write size:             4194304

8-
Commands I used (Pool size was updated appropriately):

8+
Bandwidth (MB/sec):     583.652 

9
Stddev Bandwidth:       269.76

10-
rados -p performance_test bench 120 write -b 8192 -t 100 --no-cleanup

10+
Max bandwidth (MB/sec): 884

11-
rados -p performance_test bench 120 seq -t 100

11+
Min bandwidth (MB/sec): 0

12
Average Latency:        0.68418

13
Stddev Latency:         0.552344

14
Max latency:            5.06959

15
Min latency:            0.121746

16
17
18
# rados -p performance_test bench 120 seq -b 4194304 -t 100

19
 Total time run:        58.451831

20
Total reads made:     17529

21
Read size:            4194404

22
Bandwidth (MB/sec):    1199.552 

23
Average Latency:       0.332538

24
Max latency:           3.72943

25
Min latency:           0.007074

26
27
28
1) Disk tests - All have similar numbers:

29
# dd if=/dev/zero of=here bs=1G count=1 oflag=direct

30
1+0 records in

31
1+0 records out

32
1073741824 bytes (1.1 GB) copied, 10.0691 s, 107 MB/s

33
34
2) 10G network is not holding up

35
# iperf -c 10.13.10.15  -i2 -t 10

36
37
------------------------------------------------------------

38
39
Client connecting to 10.13.10.15, TCP port 5001

40
41
TCP window size: 85.0 KByte (default)

42
43
------------------------------------------------------------

44
45
[  3] local 10.13.30.13 port 56459 connected with 10.13.10.15 port 5001

46
47
[ ID] Interval       Transfer     Bandwidth

48
49
[  3]  0.0- 2.0 sec  2.17 GBytes  9.33 Gbits/sec

50
51
[  3]  2.0- 4.0 sec  2.18 GBytes  9.37 Gbits/sec

52
53
[  3]  4.0- 6.0 sec  2.18 GBytes  9.37 Gbits/sec

54
55
[  3]  6.0- 8.0 sec  2.18 GBytes  9.38 Gbits/sec

56
57
[  3]  8.0-10.0 sec  2.18 GBytes  9.37 Gbits/sec

58
59
[  3]  0.0-10.0 sec  10.9 GBytes  9.36 Gbits/sec

60
61
62
63
3) Ceph Status

64
65
# ceph health

66
HEALTH_OK

67
root@us1-r04u05s01-ceph:~# ceph status

68
    cluster 5e95b6fa-0b99-4c31-8aa9-7a88h6hc5eda

69
     health HEALTH_OK

70
     monmap e4: 4 mons at {us1-r01u05s01-ceph=10.1.30.10:6789/0,us1-r01u09s01-ceph=10.1.30.11:6789/0,us1-r04u05s01-ceph=10.1.30.14:6789/0,us1-r04u09s01-ceph=10.1.30.15:6789/0}, election epoch 78, quorum 0,1,2,3 us1-r01u05s01-ceph,us1-r01u09s01-ceph,us1-r04u05s01-ceph,us1-r04u09s01-ceph

71
     osdmap e1029: 97 osds: 97 up, 97 in

72
      pgmap v1850869: 12480 pgs, 6 pools, 587 GB data, 116 kobjects

73
            1787 GB used, 318 TB / 320 TB avail

74
               12480 active+clean

75
  client io 0 B/s rd, 25460 B/s wr, 20 op/s

76
77
78
4) Ceph configuration

79
# cat ceph.conf 

80
81
[global]

82
  auth cluster required = cephx

83
  auth service required = cephx

84
  auth client required = cephx

85
  cephx require signatures = True

86
  cephx cluster require signatures = True

87
  cephx service require signatures = False

88
  fsid = 5e95b6fa-0b99-4c31-8aa9-7a88h6hc5eda

89
  osd pool default pg num = 4096

90
  osd pool default pgp num = 4096

91
  osd pool default size = 3

92
  osd pool default min size = 1

93
  osd pool default crush rule = 0

94
  # Disable in-memory logs

95
  debug_lockdep = 0/0

96
  debug_context = 0/0

97
  debug_crush = 0/0

98
  debug_buffer = 0/0

99
  debug_timer = 0/0

100
  debug_filer = 0/0

101
  debug_objecter = 0/0

102
  debug_rados = 0/0

103
  debug_rbd = 0/0

104
  debug_journaler = 0/0

105
  debug_objectcatcher = 0/0

106
  debug_client = 0/0

107
  debug_osd = 0/0

108
  debug_optracker = 0/0

109
  debug_objclass = 0/0

110
  debug_filestore = 0/0

111
  debug_journal = 0/0

112
  debug_ms = 0/0

113
  debug_monc = 0/0

114
  debug_tp = 0/0

115
  debug_auth = 0/0

116
  debug_finisher = 0/0

117
  debug_heartbeatmap = 0/0

118
  debug_perfcounter = 0/0

119
  debug_asok = 0/0

120
  debug_throttle = 0/0

121
  debug_mon = 0/0

122
  debug_paxos = 0/0

123
  debug_rgw = 0/0

124
125
[mon]

126
  mon osd down out interval = 600

127
  mon osd min down reporters = 2

128
    [mon.us1-r01u05s01-ceph]

129
    host = us1-r01u05s01-ceph

130
    mon addr = 10.1.30.10

131
      [mon.us1-r01u09s01-ceph]

132
    host = us1-r01u09s01-ceph

133
    mon addr = 10.1.30.11

134
      [mon.us1-r04u05s01-ceph]

135
    host = us1-r04u05s01-ceph

136
    mon addr = 10.1.30.14

137
      [mon.us1-r04u09s01-ceph]

138
    host = us1-r04u09s01-ceph

139
    mon addr = 10.1.30.15

140
  

141
[osd]

142
  osd mkfs type = xfs

143
  osd mkfs options xfs = -f -i size=2048

144
  osd mount options xfs = noatime

145
  osd journal size = 10000

146
  cluster_network = 10.2.0.0/16

147
  public_network = 10.1.0.0/16

148
  osd mon heartbeat interval = 30

149
  # Performance tuning

150
  filestore merge threshold = 40

151
  filestore split multiple = 8

152
  osd op threads = 8

153
  filestore op threads = 8

154
  filestore max sync interval = 5

155
  osd max scrubs = 1

156
  # Recovery tuning

157
  osd recovery max active = 5

158
  osd max backfills = 2

159
  osd recovery op priority = 2

160
  osd recovery max chunk = 8388608

161
  osd recovery threads = 1

162
  osd objectstore = filestore

163
  osd crush update on start = true

164
165
[mds]

166
167
168
169
5) Ceph OSDs/Crushmap

170
171
# ceph osd tree

172
# id	weight	type name	up/down	reweight

173
-14	5.46	root fusion_drives

174
-11	5.46		rack rack01-fusion

175
-7	2.73			host us1-r01u25s01-compf-fusion

176
89	2.73				osd.89	up	1	

177
-9	2.73			host us1-r01u23s01-compf-fusion

178
96	2.73				osd.96	up	1	

179
-13	315.2	root sata_drives

180
-10	166		rack rack01-sata

181
-2	76.44			host us1-r01u05s01-ceph

182
0	3.64				osd.0	up	1	

183
1	3.64				osd.1	up	1	

184
2	3.64				osd.2	up	1	

185
3	3.64				osd.3	up	1	

186
4	3.64				osd.4	up	1	

187
5	3.64				osd.5	up	1	

188
6	3.64				osd.6	up	1	

189
7	3.64				osd.7	up	1	

190
8	3.64				osd.8	up	1	

191
9	3.64				osd.9	up	1	

192
10	3.64				osd.10	up	1	

193
11	3.64				osd.11	up	1	

194
12	3.64				osd.12	up	1	

195
13	3.64				osd.13	up	1	

196
14	3.64				osd.14	up	1	

197
15	3.64				osd.15	up	1	

198
16	3.64				osd.16	up	1	

199
17	3.64				osd.17	up	1	

200
18	3.64				osd.18	up	1	

201
19	3.64				osd.19	up	1	

202
20	3.64				osd.20	up	1	

203
-3	76.44			host us1-r01u09s01-ceph

204
21	3.64				osd.21	up	1	

205
22	3.64				osd.22	up	1	

206
23	3.64				osd.23	up	1	

207
24	3.64				osd.24	up	1	

208
25	3.64				osd.25	up	1	

209
26	3.64				osd.26	up	1	

210
27	3.64				osd.27	up	1	

211
28	3.64				osd.28	up	1	

212
29	3.64				osd.29	up	1	

213
30	3.64				osd.30	up	1	

214
31	3.64				osd.31	up	1	

215
32	3.64				osd.32	up	1	

216
33	3.64				osd.33	up	1	

217
34	3.64				osd.34	up	1	

218
35	3.64				osd.35	up	1	

219
36	3.64				osd.36	up	1	

220
37	3.64				osd.37	up	1	

221
38	3.64				osd.38	up	1	

222
39	3.64				osd.39	up	1	

223
40	3.64				osd.40	up	1	

224
41	3.64				osd.41	up	1	

225
-6	6.54			host us1-r01u25s01-compf-sata

226
83	1.09				osd.83	up	1	

227
84	1.09				osd.84	up	1	

228
85	1.09				osd.85	up	1	

229
86	1.09				osd.86	up	1	

230
87	1.09				osd.87	up	1	

231
88	1.09				osd.88	up	1	

232
-8	6.54			host us1-r01u23s01-compf-sata

233
90	1.09				osd.90	up	1	

234
91	1.09				osd.91	up	1	

235
92	1.09				osd.92	up	1	

236
93	1.09				osd.93	up	1	

237
94	1.09				osd.94	up	1	

238
95	1.09				osd.95	up	1	

239
-12	149.2		rack rack04-sata

240
-4	72.8			host us1-r04u05s01-ceph

241
42	3.64				osd.42	up	1	

242
43	3.64				osd.43	up	1	

243
44	3.64				osd.44	up	1	

244
45	3.64				osd.45	up	1	

245
46	3.64				osd.46	up	1	

246
47	3.64				osd.47	up	1	

247
48	3.64				osd.48	up	1	

248
49	3.64				osd.49	up	1	

249
50	3.64				osd.50	up	1	

250
51	3.64				osd.51	up	1	

251
52	3.64				osd.52	up	1	

252
53	3.64				osd.53	up	1	

253
54	3.64				osd.54	up	1	

254
55	3.64				osd.55	up	1	

255
56	3.64				osd.56	up	1	

256
57	3.64				osd.57	up	1	

257
58	3.64				osd.58	up	1	

258
59	3.64				osd.59	up	1	

259
60	3.64				osd.60	up	1	

260
61	3.64				osd.61	up	1	

261
-5	76.44			host us1-r04u09s01-ceph

262
62	3.64				osd.62	up	1	

263
63	3.64				osd.63	up	1	

264
64	3.64				osd.64	up	1	

265
65	3.64				osd.65	up	1	

266
66	3.64				osd.66	up	1	

267
67	3.64				osd.67	up	1	

268
68	3.64				osd.68	up	1	

269
69	3.64				osd.69	up	1	

270
70	3.64				osd.70	up	1	

271
71	3.64				osd.71	up	1	

272
72	3.64				osd.72	up	1	

273
73	3.64				osd.73	up	1	

274
74	3.64				osd.74	up	1	

275
75	3.64				osd.75	up	1	

276
76	3.64				osd.76	up	1	

277
77	3.64				osd.77	up	1	

278
78	3.64				osd.78	up	1	

279
79	3.64				osd.79	up	1	

280
80	3.64				osd.80	up	1	

281
81	3.64				osd.81	up	1	

282
82	3.64				osd.82	up	1	

283
284
285
6) OSDs from one of the cluster nodes (rest are similar)

286
/dev/sda1                   3905109820 16741944 3888367876   1% /var/lib/ceph/osd/ceph-42

287
/dev/sdb1                   3905109820 19553976 3885555844   1% /var/lib/ceph/osd/ceph-43

288
/dev/sdc1                   3905109820 18081680 3887028140   1% /var/lib/ceph/osd/ceph-44

289
/dev/sdd1                   3905109820 19070596 3886039224   1% /var/lib/ceph/osd/ceph-45

290
/dev/sde1                   3905109820 17949284 3887160536   1% /var/lib/ceph/osd/ceph-46

291
/dev/sdf1                   3905109820 18538344 3886571476   1% /var/lib/ceph/osd/ceph-47

292
/dev/sdg1                   3905109820 17792608 3887317212   1% /var/lib/ceph/osd/ceph-48

293
/dev/sdh1                   3905109820 20910976 3884198844   1% /var/lib/ceph/osd/ceph-49

294
/dev/sdi1                   3905109820 19683208 3885426612   1% /var/lib/ceph/osd/ceph-50

295
/dev/sdj1                   3905109820 20115236 3884994584   1% /var/lib/ceph/osd/ceph-51

296
/dev/sdk1                   3905109820 19152812 3885957008   1% /var/lib/ceph/osd/ceph-52

297
/dev/sdm1                   3905109820 18701728 3886408092   1% /var/lib/ceph/osd/ceph-53

298
/dev/sdn1                   3905109820 19603536 3885506284   1% /var/lib/ceph/osd/ceph-54

299
/dev/sdo1                   3905109820 20164928 3884944892   1% /var/lib/ceph/osd/ceph-55

300
/dev/sdp1                   3905109820 19093024 3886016796   1% /var/lib/ceph/osd/ceph-56

301
/dev/sdq1                   3905109820 18699344 3886410476   1% /var/lib/ceph/osd/ceph-57

302
/dev/sdr1                   3905109820 19267068 3885842752   1% /var/lib/ceph/osd/ceph-58

303
/dev/sds1                   3905109820 19745212 3885364608   1% /var/lib/ceph/osd/ceph-59

304
/dev/sdt1                   3905109820 16321696 3888788124   1% /var/lib/ceph/osd/ceph-60

305
/dev/sdu1                   3905109820 19154884 3885954936   1% /var/lib/ceph/osd/ceph-61

306
307
308
6) Journal Files (there are TWO SSDs)

309
# parted /dev/sdy print

310
Model: ATA SanDisk SD7UB2Q5 (scsi)

311
Disk /dev/sdy: 512GB

312
Sector size (logical/physical): 512B/4096B

313
Partition Table: gpt

314
315
Number  Start   End     Size    File system  Name          Flags

316
 1      1049kB  10.5GB  10.5GB               ceph journal

317
 2      10.5GB  21.0GB  10.5GB               ceph journal

318
 3      21.0GB  31.5GB  10.5GB               ceph journal

319
 4      31.5GB  41.9GB  10.5GB               ceph journal

320
 5      41.9GB  52.4GB  10.5GB               ceph journal

321
 6      52.4GB  62.9GB  10.5GB               ceph journal

322
 7      62.9GB  73.4GB  10.5GB               ceph journal
        

    


            


                                


        

            



                
    

        Public Pastes
    

    
            

    

                    

        

    





    


    
    
    
    
    
    
    
    




    


    



                

            We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.             OK, I Understand
        

    
                

            

                
                    
                
            

            

                Not a member of Pastebin yet?

                Sign Up, it unlocks many cool features!