API tools faq
paste
View difference between Paste ID: h8iuzUQx and idCufE2s
SHOW: | | - or go back to the newest paste.
1
rados bench results:
2
3
Concurrency 	Replication size	       Write(MB/s)     Seq Read(MB/s)

4
32		1 			13.5		32.8

5
32		2			12.7		32.0

6
32		3			6.1		30.2

7
8
Commands I used (Pool size was updated appropriately):

9
10
rados -p performance_test bench 120 write -b 8192 -t 100 --no-cleanup

11
rados -p performance_test bench 120 seq -t 100

12
13
14
15
1) Disk tests - All have similar numbers:

16
# dd if=/dev/zero of=here bs=1G count=1 oflag=direct

17
1+0 records in

18
1+0 records out

19
1073741824 bytes (1.1 GB) copied, 10.0691 s, 107 MB/s

20
21
2) 10G network is not holding up

22
# iperf -c 10.13.10.15  -i2 -t 10

23
24
------------------------------------------------------------

25
26
Client connecting to 10.13.10.15, TCP port 5001

27
28
TCP window size: 85.0 KByte (default)

29
30
------------------------------------------------------------

31
32
[  3] local 10.13.30.13 port 56459 connected with 10.13.10.15 port 5001

33
34
[ ID] Interval       Transfer     Bandwidth

35
36
[  3]  0.0- 2.0 sec  2.17 GBytes  9.33 Gbits/sec

37
38
[  3]  2.0- 4.0 sec  2.18 GBytes  9.37 Gbits/sec

39
40
[  3]  4.0- 6.0 sec  2.18 GBytes  9.37 Gbits/sec

41
42
[  3]  6.0- 8.0 sec  2.18 GBytes  9.38 Gbits/sec

43
44
[  3]  8.0-10.0 sec  2.18 GBytes  9.37 Gbits/sec

45
46
[  3]  0.0-10.0 sec  10.9 GBytes  9.36 Gbits/sec

47
48
49
50
3) Ceph Status

51
52
# ceph health

53
HEALTH_OK

54
root@us1-r04u05s01-ceph:~# ceph status

55
    cluster 5e95b6fa-0b99-4c31-8aa9-7a88h6hc5eda

56
     health HEALTH_OK

57
     monmap e4: 4 mons at {us1-r01u05s01-ceph=10.1.30.10:6789/0,us1-r01u09s01-ceph=10.1.30.11:6789/0,us1-r04u05s01-ceph=10.1.30.14:6789/0,us1-r04u09s01-ceph=10.1.30.15:6789/0}, election epoch 78, quorum 0,1,2,3 us1-r01u05s01-ceph,us1-r01u09s01-ceph,us1-r04u05s01-ceph,us1-r04u09s01-ceph

58
     osdmap e1029: 97 osds: 97 up, 97 in

59
      pgmap v1850869: 12480 pgs, 6 pools, 587 GB data, 116 kobjects

60
            1787 GB used, 318 TB / 320 TB avail

61
               12480 active+clean

62
  client io 0 B/s rd, 25460 B/s wr, 20 op/s

63
64
65
4) Ceph configuration

66
# cat ceph.conf 

67
68
[global]

69
  auth cluster required = cephx

70
  auth service required = cephx

71
  auth client required = cephx

72
  cephx require signatures = True

73
  cephx cluster require signatures = True

74
  cephx service require signatures = False

75
  fsid = 5e95b6fa-0b99-4c31-8aa9-7a88h6hc5eda

76
  osd pool default pg num = 4096

77
  osd pool default pgp num = 4096

78
  osd pool default size = 3

79
  osd pool default min size = 1

80
  osd pool default crush rule = 0

81
  # Disable in-memory logs

82
  debug_lockdep = 0/0

83
  debug_context = 0/0

84
  debug_crush = 0/0

85
  debug_buffer = 0/0

86
  debug_timer = 0/0

87
  debug_filer = 0/0

88
  debug_objecter = 0/0

89
  debug_rados = 0/0

90
  debug_rbd = 0/0

91
  debug_journaler = 0/0

92
  debug_objectcatcher = 0/0

93
  debug_client = 0/0

94
  debug_osd = 0/0

95
  debug_optracker = 0/0

96
  debug_objclass = 0/0

97
  debug_filestore = 0/0

98
  debug_journal = 0/0

99
  debug_ms = 0/0

100
  debug_monc = 0/0

101
  debug_tp = 0/0

102
  debug_auth = 0/0

103
  debug_finisher = 0/0

104
  debug_heartbeatmap = 0/0

105
  debug_perfcounter = 0/0

106
  debug_asok = 0/0

107
  debug_throttle = 0/0

108
  debug_mon = 0/0

109
  debug_paxos = 0/0

110
  debug_rgw = 0/0

111
112
[mon]

113
  mon osd down out interval = 600

114
  mon osd min down reporters = 2

115
    [mon.us1-r01u05s01-ceph]

116
    host = us1-r01u05s01-ceph

117
    mon addr = 10.1.30.10

118
      [mon.us1-r01u09s01-ceph]

119
    host = us1-r01u09s01-ceph

120
    mon addr = 10.1.30.11

121
      [mon.us1-r04u05s01-ceph]

122
    host = us1-r04u05s01-ceph

123
    mon addr = 10.1.30.14

124
      [mon.us1-r04u09s01-ceph]

125
    host = us1-r04u09s01-ceph

126
    mon addr = 10.1.30.15

127
  

128
[osd]

129
  osd mkfs type = xfs

130
  osd mkfs options xfs = -f -i size=2048

131
  osd mount options xfs = noatime

132
  osd journal size = 10000

133
  cluster_network = 10.2.0.0/16

134
  public_network = 10.1.0.0/16

135
  osd mon heartbeat interval = 30

136
  # Performance tuning

137
  filestore merge threshold = 40

138
  filestore split multiple = 8

139
  osd op threads = 8

140
  filestore op threads = 8

141
  filestore max sync interval = 5

142
  osd max scrubs = 1

143
  # Recovery tuning

144
  osd recovery max active = 5

145
  osd max backfills = 2

146
  osd recovery op priority = 2

147
  osd recovery max chunk = 8388608

148
  osd recovery threads = 1

149
  osd objectstore = filestore

150
  osd crush update on start = true

151
152-
[mds]
152+
[mds]

153
154
155
156
5) Ceph OSDs/Crushmap

157
158
# ceph osd tree

159
# id	weight	type name	up/down	reweight

160
-14	5.46	root fusion_drives

161
-11	5.46		rack rack01-fusion

162
-7	2.73			host us1-r01u25s01-compf-fusion

163
89	2.73				osd.89	up	1	

164
-9	2.73			host us1-r01u23s01-compf-fusion

165
96	2.73				osd.96	up	1	

166
-13	315.2	root sata_drives

167
-10	166		rack rack01-sata

168
-2	76.44			host us1-r01u05s01-ceph

169
0	3.64				osd.0	up	1	

170
1	3.64				osd.1	up	1	

171
2	3.64				osd.2	up	1	

172
3	3.64				osd.3	up	1	

173
4	3.64				osd.4	up	1	

174
5	3.64				osd.5	up	1	

175
6	3.64				osd.6	up	1	

176
7	3.64				osd.7	up	1	

177
8	3.64				osd.8	up	1	

178
9	3.64				osd.9	up	1	

179
10	3.64				osd.10	up	1	

180
11	3.64				osd.11	up	1	

181
12	3.64				osd.12	up	1	

182
13	3.64				osd.13	up	1	

183
14	3.64				osd.14	up	1	

184
15	3.64				osd.15	up	1	

185
16	3.64				osd.16	up	1	

186
17	3.64				osd.17	up	1	

187
18	3.64				osd.18	up	1	

188
19	3.64				osd.19	up	1	

189
20	3.64				osd.20	up	1	

190
-3	76.44			host us1-r01u09s01-ceph

191
21	3.64				osd.21	up	1	

192
22	3.64				osd.22	up	1	

193
23	3.64				osd.23	up	1	

194
24	3.64				osd.24	up	1	

195
25	3.64				osd.25	up	1	

196
26	3.64				osd.26	up	1	

197
27	3.64				osd.27	up	1	

198
28	3.64				osd.28	up	1	

199
29	3.64				osd.29	up	1	

200
30	3.64				osd.30	up	1	

201
31	3.64				osd.31	up	1	

202
32	3.64				osd.32	up	1	

203
33	3.64				osd.33	up	1	

204
34	3.64				osd.34	up	1	

205
35	3.64				osd.35	up	1	

206
36	3.64				osd.36	up	1	

207
37	3.64				osd.37	up	1	

208
38	3.64				osd.38	up	1	

209
39	3.64				osd.39	up	1	

210
40	3.64				osd.40	up	1	

211
41	3.64				osd.41	up	1	

212
-6	6.54			host us1-r01u25s01-compf-sata

213
83	1.09				osd.83	up	1	

214
84	1.09				osd.84	up	1	

215
85	1.09				osd.85	up	1	

216
86	1.09				osd.86	up	1	

217
87	1.09				osd.87	up	1	

218
88	1.09				osd.88	up	1	

219
-8	6.54			host us1-r01u23s01-compf-sata

220
90	1.09				osd.90	up	1	

221
91	1.09				osd.91	up	1	

222
92	1.09				osd.92	up	1	

223
93	1.09				osd.93	up	1	

224
94	1.09				osd.94	up	1	

225
95	1.09				osd.95	up	1	

226
-12	149.2		rack rack04-sata

227
-4	72.8			host us1-r04u05s01-ceph

228
42	3.64				osd.42	up	1	

229
43	3.64				osd.43	up	1	

230
44	3.64				osd.44	up	1	

231
45	3.64				osd.45	up	1	

232
46	3.64				osd.46	up	1	

233
47	3.64				osd.47	up	1	

234
48	3.64				osd.48	up	1	

235
49	3.64				osd.49	up	1	

236
50	3.64				osd.50	up	1	

237
51	3.64				osd.51	up	1	

238
52	3.64				osd.52	up	1	

239
53	3.64				osd.53	up	1	

240
54	3.64				osd.54	up	1	

241
55	3.64				osd.55	up	1	

242
56	3.64				osd.56	up	1	

243
57	3.64				osd.57	up	1	

244
58	3.64				osd.58	up	1	

245
59	3.64				osd.59	up	1	

246
60	3.64				osd.60	up	1	

247
61	3.64				osd.61	up	1	

248
-5	76.44			host us1-r04u09s01-ceph

249
62	3.64				osd.62	up	1	

250
63	3.64				osd.63	up	1	

251
64	3.64				osd.64	up	1	

252
65	3.64				osd.65	up	1	

253
66	3.64				osd.66	up	1	

254
67	3.64				osd.67	up	1	

255
68	3.64				osd.68	up	1	

256
69	3.64				osd.69	up	1	

257
70	3.64				osd.70	up	1	

258
71	3.64				osd.71	up	1	

259
72	3.64				osd.72	up	1	

260
73	3.64				osd.73	up	1	

261
74	3.64				osd.74	up	1	

262
75	3.64				osd.75	up	1	

263
76	3.64				osd.76	up	1	

264
77	3.64				osd.77	up	1	

265
78	3.64				osd.78	up	1	

266
79	3.64				osd.79	up	1	

267
80	3.64				osd.80	up	1	

268
81	3.64				osd.81	up	1	

269
82	3.64				osd.82	up	1	

270
271
272
6) OSDs from one of the cluster nodes (rest are similar)

273
/dev/sda1                   3905109820 16741944 3888367876   1% /var/lib/ceph/osd/ceph-42

274
/dev/sdb1                   3905109820 19553976 3885555844   1% /var/lib/ceph/osd/ceph-43

275
/dev/sdc1                   3905109820 18081680 3887028140   1% /var/lib/ceph/osd/ceph-44

276
/dev/sdd1                   3905109820 19070596 3886039224   1% /var/lib/ceph/osd/ceph-45

277
/dev/sde1                   3905109820 17949284 3887160536   1% /var/lib/ceph/osd/ceph-46

278
/dev/sdf1                   3905109820 18538344 3886571476   1% /var/lib/ceph/osd/ceph-47

279
/dev/sdg1                   3905109820 17792608 3887317212   1% /var/lib/ceph/osd/ceph-48

280
/dev/sdh1                   3905109820 20910976 3884198844   1% /var/lib/ceph/osd/ceph-49

281
/dev/sdi1                   3905109820 19683208 3885426612   1% /var/lib/ceph/osd/ceph-50

282
/dev/sdj1                   3905109820 20115236 3884994584   1% /var/lib/ceph/osd/ceph-51

283
/dev/sdk1                   3905109820 19152812 3885957008   1% /var/lib/ceph/osd/ceph-52

284
/dev/sdm1                   3905109820 18701728 3886408092   1% /var/lib/ceph/osd/ceph-53

285
/dev/sdn1                   3905109820 19603536 3885506284   1% /var/lib/ceph/osd/ceph-54

286
/dev/sdo1                   3905109820 20164928 3884944892   1% /var/lib/ceph/osd/ceph-55

287
/dev/sdp1                   3905109820 19093024 3886016796   1% /var/lib/ceph/osd/ceph-56

288
/dev/sdq1                   3905109820 18699344 3886410476   1% /var/lib/ceph/osd/ceph-57

289
/dev/sdr1                   3905109820 19267068 3885842752   1% /var/lib/ceph/osd/ceph-58

290
/dev/sds1                   3905109820 19745212 3885364608   1% /var/lib/ceph/osd/ceph-59

291
/dev/sdt1                   3905109820 16321696 3888788124   1% /var/lib/ceph/osd/ceph-60

292
/dev/sdu1                   3905109820 19154884 3885954936   1% /var/lib/ceph/osd/ceph-61

293
294
295
6) Journal Files (there are TWO SSDs)

296
# parted /dev/sdy print

297
Model: ATA SanDisk SD7UB2Q5 (scsi)

298
Disk /dev/sdy: 512GB

299
Sector size (logical/physical): 512B/4096B

300
Partition Table: gpt

301
302
Number  Start   End     Size    File system  Name          Flags

303
 1      1049kB  10.5GB  10.5GB               ceph journal

304
 2      10.5GB  21.0GB  10.5GB               ceph journal

305
 3      21.0GB  31.5GB  10.5GB               ceph journal

306
 4      31.5GB  41.9GB  10.5GB               ceph journal

307
 5      41.9GB  52.4GB  10.5GB               ceph journal

308
 6      52.4GB  62.9GB  10.5GB               ceph journal

309
 7      62.9GB  73.4GB  10.5GB               ceph journal
        

    


            


                                


        

            



                
    

        Public Pastes
    

    
            

    

                    

        

    





    


    
    
    
    
    
    
    
    




    


    



                

            We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.             OK, I Understand
        

    
                

            

                
                    
                
            

            

                Not a member of Pastebin yet?

                Sign Up, it unlocks many cool features!