# rados -p data bench 60 write 60 -t 1
Maintaining 1 concurrent writes of 4194304 bytes for at least 60 seconds.
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
    0       0         0         0         0         0         -         0
    1       1        10         9   35.9563        36  0.096779  0.106712
    2       1        18        17   33.9777        32  0.156194  0.114834
    3       1        26        25   33.3176        32  0.102548  0.117846
    4       1        35        34   33.9872        36  0.110556   0.11501
    5       1        45        44   35.1887        40  0.105252  0.112197
    6       1        54        53   35.3233        36  0.099733  0.112842
    7       1        61        60   34.2769        28  0.161485  0.114314
    8       1        71        70   34.9917        40  0.159585  0.114145
    9       1        80        79   35.1033        36  0.093493  0.113309
   10       1        88        87   34.7927        32  0.096929  0.113729
   11       1        95        94   34.1749        28  0.119233  0.115447
   12       1       104       103   34.3267        36  0.098635  0.116034
   13       1       112       111   34.1475        32  0.110607  0.115985
   14       1       121       120   34.2796        36  0.101506  0.115957
   15       1       130       129    34.394        36  0.114899  0.116224
   16       1       138       137   34.2442        32  0.121821  0.116522
   17       1       146       145    34.112        32  0.122449  0.116764
   18       1       155       154   34.2167        36  0.155393  0.116278
   19       1       164       163   34.3103        36  0.113816   0.11646
min lat: 0.09169 max lat: 0.173117 avg lat: 0.116042
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   20       1       173       172   34.3946        36  0.097179  0.116042
   21       1       181       180   34.2804        32  0.099464  0.116174
   22       1       191       190   34.5402        40  0.097218  0.115408
   23       1       198       197   34.2558        28  0.167859  0.116207
   24       1       207       206   34.3284        36  0.162197  0.116468
   25       1       216       215   34.3951        36  0.092314  0.116278
   26       1       224       223   34.3015        32  0.100069   0.11634
   27       1       233       232   34.3643        36  0.152448  0.116184
   28       1       241       240   34.2797        32  0.160189  0.116476
   29       1       249       248    34.201        32   0.17193  0.116835
   30       1       257       256   34.1276        32  0.091647  0.117026
   31       1       265       264   34.0588        32  0.114967  0.117354
   32       1       273       272   33.9944        32   0.10182  0.117524
   33       1       281       280   33.9339        32  0.165079  0.117574
   34       1       289       288   33.8769        32  0.098787  0.117707
   35       1       298       297   33.9374        36  0.106861  0.117844
   36       1       307       306   33.9946        36  0.096477  0.117592
   37       1       315       314   33.9406        32  0.108986  0.117546
   38       1       324       323   33.9947        36  0.157954  0.117596
   39       1       332       331   33.9435        32  0.099727  0.117691
min lat: 0.091049 max lat: 0.180543 avg lat: 0.117565
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   40       1       341       340   33.9948        36   0.11237  0.117565
   41       1       350       349   34.0436        36  0.109761  0.117296
   42       1       359       358   34.0901        36  0.178623  0.117304
   43       1       368       367   34.1345        36  0.102299  0.117158
   44       1       377       376   34.1768        36  0.094891  0.116858
   45       1       386       385   34.2172        36  0.099047  0.116782
   46       1       395       394   34.2559        36   0.09911  0.116613
   47       1       404       403   34.2929        36  0.121783  0.116552
   48       1       412       411   34.2451        32  0.172545  0.116796
   49       1       420       419   34.1992        32   0.15644  0.116847
   50       1       427       426   34.0752        28  0.156228   0.11724
   51       1       436       435   34.1129        36  0.095294  0.117211
   52       1       445       444   34.1491        36  0.109894  0.117072
   53       1       453       452   34.1085        32  0.111442  0.117259
   54       1       461       460   34.0694        32  0.117764  0.117323
   55       1       468       467    33.959        28  0.109676   0.11763
   56       1       477       476   33.9954        36  0.114578  0.117529
   57       1       487       486   34.1006        40  0.099298  0.117239
   58       1       494       493   33.9954        28  0.101959  0.117477
   59       1       503       502   34.0293        36  0.128933  0.117369
min lat: 0.091049 max lat: 0.180543 avg lat: 0.1171
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   60       1       513       512   34.1287        40  0.097091    0.1171
Total time run:        60.148816
Total writes made:     514
Write size:            4194304
Bandwidth (MB/sec):    34.182

Average Latency:       0.117017
Max latency:           0.180543
Min latency:           0.091049


# rados -p data bench 60 write 60 -t 16
Maintaining 16 concurrent writes of 4194304 bytes for at least 60 seconds.
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
    0       0         0         0         0         0         -         0
    1      16        40        24   95.9864        96  0.501944  0.492511
    2      16        69        53   105.987       116  0.598758   0.53294
    3      16        96        80   106.655       108  0.226725  0.544784
    4      16       120       104   103.989        96  0.141996  0.520572
    5      16       148       132   105.589       112   1.02541  0.541458
    6      16       174       158   105.322       104  0.256355  0.564897
    7      16       203       187   106.846       116  0.168386  0.562557
    8      16       226       210   104.989        92  0.324106  0.576739
    9      16       250       234   103.989        96  0.484063  0.574434
   10      16       264       248   99.1896        56   1.44125  0.595992
   11      16       293       277   100.717       116  0.665886  0.619594
   12      16       320       304   101.323       108  0.997471  0.615557
   13      16       339       323   99.3743        76  0.990265  0.619465
   14      16       369       353   100.847       120  0.413741  0.624125
   15      16       393       377   100.523        96  0.140511  0.612518
   16      16       415       399   99.7398        88  0.139286  0.619995
   17      16       445       429   100.931       120  0.170344  0.609091
   18      16       474       458   101.767       116  0.185543  0.611482
   19      16       502       486   102.305       112  0.595428  0.614518
min lat: 0.097672 max lat: 2.09864 avg lat: 0.610924
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   20      16       524       508   101.589        88  0.233334  0.610924
   21      16       552       536   102.085       112  0.862635  0.616295
   22      16       577       561    101.99       100  0.205261  0.615365
   23      16       609       593    103.12       128   0.25302  0.609307
   24      16       633       617   102.823        96  0.611891  0.609097
   25      16       663       647   103.509       120   0.44431  0.612382
   26      16       686       670   103.066        92  0.333727  0.608059
   27      16       712       696   103.101       104  0.215208  0.604521
   28      16       731       715   102.132        76  0.688289  0.617328
   29      16       763       747   103.024       128  0.627129  0.614417
   30      16       791       775   103.323       112  0.202008  0.609439
   31      16       811       795    102.57        80   1.54344   0.61149
   32      16       840       824   102.989       116    0.2819   0.61638
   33      16       867       851   103.119       108  0.142422  0.611737
   34      16       896       880   103.498       116  0.238483  0.605784
   35      16       915       899   102.712        76  0.150481  0.603014
   36      16       943       927    102.97       112  0.214514  0.615291
   37      16       971       955   103.213       112  0.720115  0.615043
   38      16       995       979   103.023        96   1.07134  0.612726
   39      16      1023      1007   103.253       112  0.302163    0.6128
min lat: 0.09461 max lat: 2.27008 avg lat: 0.612471
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   40      16      1054      1038   103.771       124  0.543701  0.612471
   41      16      1074      1058   103.191        80  0.765966  0.612676
   42      16      1102      1086   103.401       112  0.113156  0.608052
   43      16      1128      1112   103.415       104  0.509544  0.611522
   44      16      1143      1127   102.428        60  0.893179  0.617971
   45      16      1168      1152   102.374       100  0.163758  0.617595
   46      16      1196      1180   102.583       112   1.20921  0.617989
   47      16      1220      1204   102.442        96  0.363124  0.619866
   48      16      1251      1235   102.891       124   1.42413  0.615254
   49      16      1278      1262   102.995       108   1.73097  0.613579
   50      16      1306      1290   103.175       112  0.663365  0.616592
   51      16      1317      1301   102.015        44   0.38472  0.614968
   52      16      1346      1330   102.284       116  0.701459  0.621164
   53      16      1372      1356   102.316       104  0.323221  0.621663
   54      16      1398      1382   102.347       104  0.199313  0.620405
   55      16      1425      1409   102.449       108  0.134548  0.616961
   56      16      1452      1436   102.548       108    0.3728  0.620143
   57      16      1481      1465   102.784       116  0.155109  0.615704
   58      16      1502      1486    102.46        84  0.549346  0.620348
   59      16      1530      1514   102.622       112  0.235383  0.619207
min lat: 0.09461 max lat: 2.27008 avg lat: 0.617439
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   60      16      1562      1546   103.044       128  0.549766  0.617439
Total time run:        60.575964
Total writes made:     1563
Write size:            4194304
Bandwidth (MB/sec):    103.209

Average Latency:       0.618547
Max latency:           2.27008
Min latency:           0.09461


# rados -p data bench 60 seq -t 1
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
    0       0         0         0         0         0         -         0
    1       1        20        19   75.9878        76  0.046893 0.0519567
    2       1        41        40   79.9896        84  0.059661 0.0497915
    3       1        60        59   78.6574        76  0.045953 0.0502848
    4       1        80        79   78.9911        80  0.050066 0.0505887
    5       1        99        98   78.3914        76  0.047703 0.0509201
    6       1       119       118   78.6583        80   0.04715 0.0506965
    7       1       139       138   78.8488        80   0.06137 0.0504733
    8       1       160       159   79.4916        84  0.047615 0.0501425
    9       1       180       179   79.5466        80   0.04601 0.0502078
   10       1       198       197   78.7913        72  0.062198 0.0505328
   11       1       218       217   78.9005        80  0.044672 0.0505325
   12       1       239       238   79.3249        84   0.04579  0.050413
   13       1       257       256   78.7609        72  0.044867 0.0506867
   14       1       275       274   78.2775        72  0.046132 0.0509453
   15       1       295       294   78.3918        80  0.062158 0.0508668
   16       1       315       314   78.4919        80  0.061731 0.0509351
   17       1       335       334   78.5802        80  0.044794 0.0508101
   18       1       355       354   78.6586        80  0.047019  0.050755
   19       1       375       374   78.7288        80  0.062165 0.0506912
min lat: 0.044035 max lat: 0.076209 avg lat: 0.0507382
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   20       1       395       394    78.792        80  0.047527 0.0507382
   21       1       415       414   78.8492        80  0.062031 0.0506667
   22       1       435       434   78.9011        80  0.047543 0.0506158
   23       1       455       454   78.9488        80  0.045836 0.0506609
   24       1       474       473   78.8256        76  0.045749 0.0506348
   25       1       495       494   79.0323        84  0.044857 0.0505215
   26       1       517       516   79.3768        88  0.047517 0.0503833
   27       1       536       535   79.2515        76  0.046026 0.0504586
   28       1       554       553   78.9923        72  0.062082 0.0506166
   29       1       573       572   78.8888        76  0.046884 0.0506588
   30       1       593       592   78.9256        80  0.046771 0.0506728
   31       1       612       611    78.831        76  0.046337 0.0507214
   32       1       633       632   78.9923        84  0.044514 0.0506151
   33       1       654       653   79.1437        84   0.04673 0.0505039
   34       1       675       674   79.2863        84  0.047768 0.0503866
   35       1       697       696    79.535        88   0.04619 0.0502768
   36       1       718       717   79.6588        84   0.04735 0.0501784
   37       1       739       738   79.7759        84  0.047117  0.050093
   38       1       760       759   79.8869        84  0.053935 0.0500429
   39       1       781       780   79.9921        84   0.04509 0.0499661
min lat: 0.044035 max lat: 0.076209 avg lat: 0.0499152
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   40       1       802       801   80.0919        84   0.04751 0.0499152
   41       1       823       822   80.1871        84   0.04596  0.049832
   42       1       844       843   80.2777        84  0.051334  0.049771
   43       1       866       865    80.457        88   0.04396 0.0496929
   44       1       887       886   80.5373        84   0.04744 0.0496406
   45       1       908       907    80.614        84  0.046054 0.0495911
   46       1       929       928   80.6875        84   0.04551 0.0495305
   47       1       950       949   80.7578        84  0.045918 0.0494886
   48       1       971       970   80.8251        84  0.044626 0.0494414
   49       1       993       992   80.9714        88  0.047375 0.0493933
   50       1      1014      1013   81.0318        84  0.044178 0.0493372
   51       1      1035      1034   81.0898        84   0.04673 0.0492921
   52       1      1057      1056   81.2225        88  0.047269 0.0492429
   53       1      1078      1077   81.2748        84  0.045742 0.0492021
   54       1      1099      1098   81.3251        84  0.046806 0.0491561
   55       1      1120      1119   81.3736        84  0.047617 0.0491193
   56       1      1141      1140   81.4203        84  0.047016 0.0490879
   57       1      1162      1161   81.4654        84  0.047478 0.0490856
   58       1      1183      1182    81.509        84  0.048089 0.0490631
   59       1      1204      1203   81.5511        84  0.054935 0.0490237
min lat: 0.043575 max lat: 0.076209 avg lat: 0.0489954
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   60       1      1225      1224   81.5917        84  0.047442 0.0489954
Total time run:        60.073449
Total reads made:     1226
Read size:            4194304
Bandwidth (MB/sec):    81.633

Average Latency:       0.0489916
Max latency:           0.076209
Min latency:           0.043575



# rados -p data bench 60 seq -t 16
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
    0       0         0         0         0         0         -         0
    1      16        69        53   211.966       212  0.410599  0.258549
    2      16       123       107   213.975       216  0.077392  0.258418
    3      16       175       159   211.977       208  0.278645  0.267079
    4      16       221       205   204.978       184  0.776925  0.282649
    5      16       265       249   199.179       176  0.685209   0.30242
    6      16       314       298   198.646       196  0.084217  0.301528
    7      16       368       352   201.122       216   0.07471  0.306663
    8      16       420       404   201.979       208  0.181303   0.30249
    9      16       474       458   203.534       216  0.086136  0.302854
   10      16       521       505   201.979       188   1.01302  0.302879
   11      16       566       550   199.979       180  0.065868  0.308581
   12      16       611       595   198.313       180  0.092272  0.315165
   13      16       657       641   197.211       184  0.875013    0.3153
   14      16       701       685   195.694       176  0.935379  0.317712
   15      16       753       737   196.513       208  0.049284  0.319039
   16      16       800       784    195.98       188  0.892173  0.319196
   17      16       849       833    195.98       196  0.058948  0.319177
   18      16       898       882    195.98       196  0.095366  0.320581
   19      16       942       926   194.927       176  0.089717  0.322121
min lat: 0.043517 max lat: 1.08818 avg lat: 0.322654
  sec Cur ops   started  finished  avg MB/s  cur MB/s  last lat   avg lat
   20      16       998       982    196.38       224  0.172038  0.322654
   21      16      1052      1036   197.313       216  0.378776  0.321404
   22      16      1107      1091   198.343       220   0.39724  0.320037
   23      16      1162      1146   199.284       220  0.645085  0.317653
   24      16      1219      1203    200.48       228  0.247228  0.316728
   25      16      1273      1257     201.1       216  0.538584  0.315984
   26      16      1328      1312   201.826       220  0.212595  0.315263
   27      16      1385      1369   202.794       228  0.161096  0.313519
   28      16      1429      1413   201.837       176  0.647564  0.314685
   29      16      1481      1465   202.049       208  0.152063  0.315167
   30      16      1535      1519   202.513       216  0.414549  0.314017
Total time run:        30.956005
Total reads made:     1563
Read size:            4194304
Bandwidth (MB/sec):    201.964

Average Latency:       0.315992
Max latency:           1.08818
Min latency:           0.043517