test script - mdadm ext4 vs btrfs, fio ssd-test 4k and 256k

#!/bin/bash
#
# Author: Peter Maloney
# License: GPLv2
#
# A benchmark script for testing many different layouts with 4 disks.
# It is a work in progress.
# Much of it is hardcoded... eg. 4 disks
# It doesn't test different io schedulers.
#
# To use:
#     cd to the directory where you want the logs:
#         mkdir ~/raidtest1/
#         cd ~/raidtest1/
#     then run the script in test mode:
#         ~/path/to/raidtest.bash test
#     then run the script in either results or csv mode:
#         ~/path/to/raidtest.bash results
#         ~/path/to/raidtest.bash csv

testd1=/dev/sda2
testd2=/dev/sdb2
testd3=/dev/sdc2
testd4=/dev/sdd2
testmd=/dev/md127

btrfspath="/home/peter/projects/btrfs-progs"

fio=/home/peter/archive/software/fio/fio-2.0.10/fio

testConfigFile4k="/home/peter/archive/software/fio/fio-2.0.10/examples/ssd-test-short"
testConfigFile256k="/home/peter/archive/software/fio/fio-2.0.10/examples/ssd-test-short256k"

if [ "$#" -lt 1 -o "$#" -gt 2 ]; then
    echo "USAGE: $0 <mode> [configFile]"
    echo "    modes are: test, results, csv"
    echo "    configFile is either: 4k, or 256k (default), or a path to an fio test config file"
    exit 1
fi

mode="$1"
testConfigFile="$2"

if [ "$mode" = "test" ]; then
    if [ "$testConfigFile" = "4k" ]; then
        testConfigFile="$testConfigFile4k"
    elif [ "$testConfigFile" = "256k" ]; then
        testConfigFile="$testConfigFile256k"
    elif [ -f "$testConfigFile" ]; then
        echo "Using custom config file: $testConfigFile"
    else
        testConfigFile="$testConfigFile256k"
        echo "Using default config file: $testConfigFile"
    fi
fi

cleanTest() {
    if mountpoint /mnt/TestDevice >/dev/null; then
        umount /mnt/TestDevice
    fi
    if [ -e "$testmd" ]; then
        mdadm --stop "${testmd}"
        wipefs -a "$testmd"
    fi
    mdadm --zero-superblock "$testd1" 2>/dev/null
    mdadm --zero-superblock "$testd2" 2>/dev/null
    mdadm --zero-superblock "$testd3" 2>/dev/null
    mdadm --zero-superblock "$testd4" 2>/dev/null
    wipefs -a "$testd1" 2>/dev/null
    wipefs -a "$testd2" 2>/dev/null
    wipefs -a "$testd3" 2>/dev/null
    wipefs -a "$testd4" 2>/dev/null
    rmdir /mnt/TestDevice 2>/dev/null
}

runTest() {
    fs="$1"
    level="$2"
    disks="$3"

    pwd=$(pwd)
    if ! mountpoint /mnt/TestDevice; then
        echo "ERROR: /mnt/TestDevice is not a mountpoint"
        return 1
    fi

    cd /mnt/TestDevice
        "$fio" \
        "$testConfigFile" \
        | tee "$pwd/$fs $level $disks".log

    # for debugging / verification purposes, just to prove the test was really testing what it should have been
    echo >> "$pwd/$fs $level $disks".log
    mdadm -D "$testmd" >> "$pwd/$fs $level $disks".log
    echo >> "$pwd/$fs $level $disks".log
    btrfs fi show TestDevice >> "$pwd/$fs $level $disks".log
    echo >> "$pwd/$fs $level $disks".log
    mount >> "$pwd/$fs $level $disks".log

    rm ssd.test.file
    cd "$pwd"
}

setupTest() {
    fs="$1"
    level="$2"
    disks="$3"

    devs=
    if [ "$disks" = "1" ]; then
        devs="$testd1"
    elif [ "$disks" = "2" ]; then
        devs="$testd1 $testd2"
    elif [ "$disks" = "3" ]; then
        devs="$testd1 $testd2 $testd3"
    elif [ "$disks" = "4" ]; then
        devs="$testd1 $testd2 $testd3 $testd4"
    fi

    if [ "$fs" = "btrfs" ]; then
        if [ "$level" = "raid10-far" ]; then
            echo "skipping test: $fs $level $disks"
            return 1
        fi
        "$btrfspath"/mkfs.btrfs -L TestDevice -m "$level" -d "$level" $devs || return 2
        mkdir /mnt/TestDevice
        mount -o noatime "$testd1" /mnt/TestDevice
    elif [ "$fs" = "mdadm ext4" -a "$level" = "single" ]; then
        mkfs.ext4 -L TestDevice -m 0 "$testd1"
        mkdir /mnt/TestDevice
        mount -o noatime "$testd1" /mnt/TestDevice
    elif [ "$fs" = "mdadm ext4" ]; then
        mlevel=
        mlayout=
        if [ "$level" = "raid10-far" ]; then
            mlevel="--level 10"
            mlayout="--layout f2"
        else
            mlevel="--level ${level:4}"
        fi

        # using --metadata=0.90 to stop the interactive "yes no" junk related to booting
        mdadm --create "$testmd" --metadata=0.90 -n "$disks" -x 0 -N "TestDevice" -W $mlevel $mlayout $devs || return 2
        mkfs.ext4 -L TestDevice -m 0 "$testmd"
        mkdir /mnt/TestDevice
        mount -o noatime "$testmd" /mnt/TestDevice

# next block obsolete by mdadm -W option
#        echo -n "Waiting for mdadm sync to complete"
#        while mdadm -D "$testmd" | grep "State.*active.*resyncing" >/dev/null; do
#            echo -n "."
#            sleep 1
#        done
        echo
    else
        echo "ERROR: bad fs: $fs"
        return 1
    fi
}

runAllTests() {
    for fs in btrfs "mdadm ext4"; do
        for level in raid0 raid1 raid10 raid10-far single; do
            for disks in {1..4}; do
                # skip tests that don't make sense
                if [ "$level" = "raid0" -a "$disks" = 1 ]; then
                    continue
                elif [ "$level" = "raid1" -a "$disks" != 2 -a '(' "$disks" != 1 -o "$fs" != "btrfs" ')' ]; then
                    # don't test btrfs raid1 with 1 disk
                    # don't test mdadm raid1 with anything except 2 disks
                    continue
                elif [ "$level" = "raid10-far" -a "$fs" = "btrfs" ]; then
                    continue
                elif [ "$level" = "raid10" -a "$disks" != 4 ]; then
                    continue
                elif [ "$level" = "single" -a "$disks" != 1 -a "$fs" != "btrfs" ]; then
                    continue
                fi

                echo "======================================"
                echo "next test: $fs $level $disks"
                echo "======================================"
                cleanTest
                setupTest "$fs" "$level" "$disks" \
                    && runTest "$fs" "$level" "$disks"
                echo
                echo
            done
        done
    done
}

testResults() {
    for t in seq-read rand-read seq-write rand-write; do
        echo "======================================"
        echo "$t"
        echo "======================================"
        grep -E "${t}: \(groupid" -A 1 *.log
    done
}

testResultsCsv() {
    #for f in "mdadm ext4 single 4.log"; do
    for f in *.log; do
        #echo "DEBUG: f is $f"

        setup=$(echo "$f" | sed -r "s/.log$//")

        IFS=$'\n'
        for t in seq-read rand-read seq-write rand-write; do
            for line in $(grep -E "${t}: \(groupid" -A 1 "$f"); do
                #echo "DEBUG: line is $line"

                first=$(echo "$line" | cut -f1 -d:)
                if [ "$first" = "seq-read" ]; then
                    test="seq-read"
                elif [ "$first" = "rand-read" ]; then
                    test="rand-read"
                elif [ "$first" = "seq-write" ]; then
                    test="seq-write"
                elif [ "$first" = "rand-write" ]; then
                    test="rand-write"
                else
                    bw=$(echo "$line" | grep -Eo "bw=[0-9\.GMKB\/s ]+" | cut -f2 -d'=' )

                    bwValue=$(echo "$bw" | grep -Eo "[0-9\.]+")
                    bwUnit=$(echo "$bw" | grep -Eo "[GMKB\/s]+")

                    # convert to KB/s
                    if [ "$bwUnit" = "GB/s" ]; then
                        bw=$(echo "$bwValue * 1000000" | bc)
                    elif [ "$bwUnit" = "MB/s" ]; then
                        bw=$(echo "$bwValue * 1000" | bc)
                    elif [ "$bwUnit" = "B/s" ]; then
                        bw=$(echo "$bwValue / 1000" | bc)
                    elif [ "$bwUnit" = "KB/s" ]; then
                        bw=$bwValue
                    else
                        echo "ERROR: unparsable bw = $bw"
                        bw=$bwValue
                    fi

                    iops=$(echo "$line" | grep -Eo "iops=[0-9]+" | cut -f2 -d'=' )
                fi
            done

            echo "$setup, $test, $bw, $iops"
        done
    done
}

if [ "$mode" = "test" ]; then
    runAllTests
elif [ "$mode" = "results" ]; then
    testResults
elif [ "$mode" = "csv" ]; then
    testResultsCsv > results.csv
    grep "seq-read" results.csv > "results.seq-read.csv"
    grep "seq-write" results.csv > "results.seq-write.csv"
    grep "rand-read" results.csv > "results.rand-read.csv"
    grep "rand-write" results.csv > "results.rand-write.csv"
fi