compression comparison

#!/bin/bash
set -e

##
##  scroll down to end of file for test results
##  measured on i5-6300HQ, 4 cores, 4 threads
##  ( with some cpu load from chrome etc )
##

infile=/dev/shm/usr-lib.tar

tests=(
    "pigz -c -0" \
    "pigz -c -1" \
    "pigz -c -2" \
    "pigz -c -3" \
    "pigz -c -4" \
    "pigz -c -5" \
    "pigz -c -6" \
    "pigz -c -7" \
    "pigz -c -8" \
    "pigz -c -9" \
    "pigz -c -9 -b 32" \
    "pigz -c -9 -b 64" \
    "pigz -c -9 -b 256" \
    "pigz -c -9 -b 512" \
    "pigz -c -9 -b 1024" \
    "lbzip2 -c -1" \
    "lbzip2 -c -2" \
    "lbzip2 -c -3" \
    "lbzip2 -c -4" \
    "lbzip2 -c -5" \
    "lbzip2 -c -6" \
    "lbzip2 -c -7" \
    "lbzip2 -c -8" \
    "lbzip2 -c -9" \
    "lbzip2 -c -9 -s" \
    "lbzip2 -c -9 -u" \
    "xz -z -c -0 -T 0" \
    "xz -z -c -1 -T 0" \
    "xz -z -c -2 -T 0" \
    "xz -z -c -3 -T 0" \
    "xz -z -c -4 -T 0" \
    "xz -z -c -5 -T 0" \
    "xz -z -c -6 -T 0" \
    "xz -z -c -7 -T 0" \
    "xz -z -c -8 -T 0" \
    "xz -z -c -9 -T 0" \
    "xz -z -c -0 -T 0 -e" \
    "xz -z -c -1 -T 0 -e" \
    "xz -z -c -2 -T 0 -e" \
    "xz -z -c -3 -T 0 -e" \
    "xz -z -c -4 -T 0 -e" \
    "xz -z -c -5 -T 0 -e" \
    "xz -z -c -6 -T 0 -e" \
    "xz -z -c -7 -T 0 -e" \
    "xz -z -c -8 -T 0 -e" \
    "xz -z -c -9 -T 0 -e"
)

paratests=(
    "xz -z -c -0" \
    "xz -z -c -1" \
    "xz -z -c -2" \
    "xz -z -c -3" \
    "xz -z -c -4" \
    "xz -z -c -5" \
    "xz -z -c -6" \
    "xz -z -c -7" \
    "xz -z -c -8" \
    "xz -z -c -9" \
    "xz -z -c -0 -e" \
    "xz -z -c -1 -e" \
    "xz -z -c -2 -e" \
    "xz -z -c -3 -e" \
    "xz -z -c -4 -e" \
    "xz -z -c -5 -e" \
    "xz -z -c -6 -e" \
    "xz -z -c -7 -e" \
    "xz -z -c -8 -e" \
    "xz -z -c -9 -e"
)

printf '%10s bytes in %s\n' \
    "$(stat -c%s $infile)" \
    "$infile"

for args in "${tests[@]}"
do
    t1=$(date +%s%N)
    sz=$($args < $infile | wc -c)
    t2=$(date +%s%N)
    td=$(
        printf '%s\n' $((t2-t1)) |
        sed -r 's/(.{3}).{6}$/.\1/'
    )
    printf '%10s bytes in %7s seconds using %s\n' $sz $td "$args"
done

cores=4
used=0
for args in "${paratests[@]}"
do
    used=$((used+1))

    (( used > cores )) &&
    {
        wait
        used=1
    }

    # forked worker
    {
        t1=$(date +%s%N)
        sz=$($args < $infile | wc -c)
        t2=$(date +%s%N)
        td=$(
            printf '%s\n' $((t2-t1)) |
            sed -r 's/(.{3}).{6}$/.\1/'
        )
        printf '%10s bytes in %7s seconds using %s\n' $sz $td "$args"
    } &
done
wait

exit 0



1505812480 bytes in /dev/shm/usr-lib.tar
1505984823 bytes in   1.969 seconds using pigz -c -0
 570615116 bytes in   7.415 seconds using pigz -c -1
 559151803 bytes in   7.502 seconds using pigz -c -2
 549957640 bytes in   9.310 seconds using pigz -c -3
 537060988 bytes in   9.703 seconds using pigz -c -4
 526524677 bytes in  12.712 seconds using pigz -c -5
 521648443 bytes in  18.680 seconds using pigz -c -6
 520409025 bytes in  23.048 seconds using pigz -c -7
 518994457 bytes in  46.394 seconds using pigz -c -8
 518574325 bytes in  75.735 seconds using pigz -c -9
 517163600 bytes in  81.157 seconds using pigz -c -9 -b 32
 518170569 bytes in  75.898 seconds using pigz -c -9 -b 64
 518574325 bytes in  75.157 seconds using pigz -c -9 -b 128
 518699699 bytes in  77.513 seconds using pigz -c -9 -b 256
 518766300 bytes in  76.649 seconds using pigz -c -9 -b 512
 518818092 bytes in  75.173 seconds using pigz -c -9 -b 1024
 505057829 bytes in  29.661 seconds using lbzip2 -c -1
 493208559 bytes in  27.124 seconds using lbzip2 -c -2
 488042181 bytes in  26.147 seconds using lbzip2 -c -3
 484855922 bytes in  25.937 seconds using lbzip2 -c -4
 482714163 bytes in  25.855 seconds using lbzip2 -c -5
 480663960 bytes in  26.019 seconds using lbzip2 -c -6
 479345986 bytes in  26.087 seconds using lbzip2 -c -7
 478339629 bytes in  26.459 seconds using lbzip2 -c -8
 477214680 bytes in  28.517 seconds using lbzip2 -c -9
 477214680 bytes in  26.333 seconds using lbzip2 -c -9 -s
 476277924 bytes in  26.375 seconds using lbzip2 -c -9 -u
 444967732 bytes in  29.076 seconds using xz -z -c -0 -T 0
 419181364 bytes in  45.030 seconds using xz -z -c -1 -T 0
 408102088 bytes in  74.835 seconds using xz -z -c -2 -T 0
 400795800 bytes in 104.774 seconds using xz -z -c -3 -T 0
 384321280 bytes in 130.551 seconds using xz -z -c -4 -T 0
 368988288 bytes in 183.542 seconds using xz -z -c -5 -T 0
 367034284 bytes in 209.514 seconds using xz -z -c -6 -T 0
 359465936 bytes in 223.646 seconds using xz -z -c -7 -T 0
 355302128 bytes in 238.301 seconds using xz -z -c -8 -T 0
 408136360 bytes in 149.856 seconds using xz -z -c -0 -T 0 -e
 389491264 bytes in 168.247 seconds using xz -z -c -1 -T 0 -e
 381135596 bytes in 178.825 seconds using xz -z -c -2 -T 0 -e
 373894068 bytes in 179.325 seconds using xz -z -c -3 -T 0 -e
 374142148 bytes in 179.662 seconds using xz -z -c -4 -T 0 -e
 366809628 bytes in 182.341 seconds using xz -z -c -5 -T 0 -e
 366947768 bytes in 192.392 seconds using xz -z -c -6 -T 0 -e
 359222592 bytes in 208.451 seconds using xz -z -c -7 -T 0 -e
 355054912 bytes in 239.716 seconds using xz -z -c -8 -T 0 -e
 440812956 bytes in 138.224 seconds using xz -z -c -0
 414591336 bytes in 167.587 seconds using xz -z -c -1
 404333924 bytes in 209.236 seconds using xz -z -c -2
 397590264 bytes in 259.024 seconds using xz -z -c -3
 380756200 bytes in 459.342 seconds using xz -z -c -4
 365909072 bytes in 587.287 seconds using xz -z -c -5
 363897956 bytes in 650.989 seconds using xz -z -c -6
 356567660 bytes in 673.724 seconds using xz -z -c -7
 352942048 bytes in 682.302 seconds using xz -z -c -8
 403088120 bytes in 617.040 seconds using xz -z -c -0 -e
 384136200 bytes in 669.630 seconds using xz -z -c -1 -e
 376549732 bytes in 695.215 seconds using xz -z -c -2 -e
 369914736 bytes in 706.825 seconds using xz -z -c -3 -e
 370125100 bytes in 740.674 seconds using xz -z -c -4 -e
 363587780 bytes in 732.890 seconds using xz -z -c -5 -e
 363778076 bytes in 767.257 seconds using xz -z -c -6 -e
 356411268 bytes in 734.169 seconds using xz -z -c -7 -e
 352687584 bytes in 764.315 seconds using xz -z -c -8 -e

# sorted by file size:
# grep -E '^ *[0-9]+ bytes' pigz-test.sh | sort -nk 1
 352687584 bytes in 764.315 seconds using xz -z -c -8 -e
 352942048 bytes in 682.302 seconds using xz -z -c -8
 355054912 bytes in 239.716 seconds using xz -z -c -8 -T 0 -e
 355302128 bytes in 238.301 seconds using xz -z -c -8 -T 0
 356411268 bytes in 734.169 seconds using xz -z -c -7 -e
 356567660 bytes in 673.724 seconds using xz -z -c -7
 359222592 bytes in 208.451 seconds using xz -z -c -7 -T 0 -e
 359465936 bytes in 223.646 seconds using xz -z -c -7 -T 0
 363587780 bytes in 732.890 seconds using xz -z -c -5 -e
 363778076 bytes in 767.257 seconds using xz -z -c -6 -e
 363897956 bytes in 650.989 seconds using xz -z -c -6
 365909072 bytes in 587.287 seconds using xz -z -c -5
 366809628 bytes in 182.341 seconds using xz -z -c -5 -T 0 -e
 366947768 bytes in 192.392 seconds using xz -z -c -6 -T 0 -e
 367034284 bytes in 209.514 seconds using xz -z -c -6 -T 0
 368988288 bytes in 183.542 seconds using xz -z -c -5 -T 0
 369914736 bytes in 706.825 seconds using xz -z -c -3 -e
 370125100 bytes in 740.674 seconds using xz -z -c -4 -e
 373894068 bytes in 179.325 seconds using xz -z -c -3 -T 0 -e
 374142148 bytes in 179.662 seconds using xz -z -c -4 -T 0 -e
 376549732 bytes in 695.215 seconds using xz -z -c -2 -e
 380756200 bytes in 459.342 seconds using xz -z -c -4
 381135596 bytes in 178.825 seconds using xz -z -c -2 -T 0 -e
 384136200 bytes in 669.630 seconds using xz -z -c -1 -e
 384321280 bytes in 130.551 seconds using xz -z -c -4 -T 0
 389491264 bytes in 168.247 seconds using xz -z -c -1 -T 0 -e
 397590264 bytes in 259.024 seconds using xz -z -c -3
 400795800 bytes in 104.774 seconds using xz -z -c -3 -T 0
 403088120 bytes in 617.040 seconds using xz -z -c -0 -e
 404333924 bytes in 209.236 seconds using xz -z -c -2
 408102088 bytes in  74.835 seconds using xz -z -c -2 -T 0
 408136360 bytes in 149.856 seconds using xz -z -c -0 -T 0 -e
 414591336 bytes in 167.587 seconds using xz -z -c -1
 419181364 bytes in  45.030 seconds using xz -z -c -1 -T 0
 440812956 bytes in 138.224 seconds using xz -z -c -0
 444967732 bytes in  29.076 seconds using xz -z -c -0 -T 0
 476277924 bytes in  26.375 seconds using lbzip2 -c -9 -u
 477214680 bytes in  26.333 seconds using lbzip2 -c -9 -s
 477214680 bytes in  28.517 seconds using lbzip2 -c -9
 478339629 bytes in  26.459 seconds using lbzip2 -c -8
 479345986 bytes in  26.087 seconds using lbzip2 -c -7
 480663960 bytes in  26.019 seconds using lbzip2 -c -6
 482714163 bytes in  25.855 seconds using lbzip2 -c -5
 484855922 bytes in  25.937 seconds using lbzip2 -c -4
 488042181 bytes in  26.147 seconds using lbzip2 -c -3
 493208559 bytes in  27.124 seconds using lbzip2 -c -2
 505057829 bytes in  29.661 seconds using lbzip2 -c -1
 517163600 bytes in  81.157 seconds using pigz -c -9 -b 32
 518170569 bytes in  75.898 seconds using pigz -c -9 -b 64
 518574325 bytes in  75.157 seconds using pigz -c -9 -b 128
 518574325 bytes in  75.735 seconds using pigz -c -9
 518699699 bytes in  77.513 seconds using pigz -c -9 -b 256
 518766300 bytes in  76.649 seconds using pigz -c -9 -b 512
 518818092 bytes in  75.173 seconds using pigz -c -9 -b 1024
 518994457 bytes in  46.394 seconds using pigz -c -8
 520409025 bytes in  23.048 seconds using pigz -c -7
 521648443 bytes in  18.680 seconds using pigz -c -6
 526524677 bytes in  12.712 seconds using pigz -c -5
 537060988 bytes in   9.703 seconds using pigz -c -4
 549957640 bytes in   9.310 seconds using pigz -c -3
 559151803 bytes in   7.502 seconds using pigz -c -2
 570615116 bytes in   7.415 seconds using pigz -c -1
1505812480 bytes in /dev/shm/usr-lib.tar
1505984823 bytes in   1.969 seconds using pigz -c -0

# sorted by processing time:
# grep -E '^ *[0-9]+ bytes' pigz-test.sh | sort -nk 4 | uniq
1505812480 bytes in /dev/shm/usr-lib.tar
1505984823 bytes in   1.969 seconds using pigz -c -0
 570615116 bytes in   7.415 seconds using pigz -c -1
 559151803 bytes in   7.502 seconds using pigz -c -2
 549957640 bytes in   9.310 seconds using pigz -c -3
 537060988 bytes in   9.703 seconds using pigz -c -4
 526524677 bytes in  12.712 seconds using pigz -c -5
 521648443 bytes in  18.680 seconds using pigz -c -6
 520409025 bytes in  23.048 seconds using pigz -c -7
 482714163 bytes in  25.855 seconds using lbzip2 -c -5
 484855922 bytes in  25.937 seconds using lbzip2 -c -4
 480663960 bytes in  26.019 seconds using lbzip2 -c -6
 479345986 bytes in  26.087 seconds using lbzip2 -c -7
 488042181 bytes in  26.147 seconds using lbzip2 -c -3
 477214680 bytes in  26.333 seconds using lbzip2 -c -9 -s
 476277924 bytes in  26.375 seconds using lbzip2 -c -9 -u
 478339629 bytes in  26.459 seconds using lbzip2 -c -8
 493208559 bytes in  27.124 seconds using lbzip2 -c -2
 477214680 bytes in  28.517 seconds using lbzip2 -c -9
 444967732 bytes in  29.076 seconds using xz -z -c -0 -T 0
 505057829 bytes in  29.661 seconds using lbzip2 -c -1
 419181364 bytes in  45.030 seconds using xz -z -c -1 -T 0
 518994457 bytes in  46.394 seconds using pigz -c -8
 408102088 bytes in  74.835 seconds using xz -z -c -2 -T 0
 518574325 bytes in  75.157 seconds using pigz -c -9 -b 128
 518818092 bytes in  75.173 seconds using pigz -c -9 -b 1024
 518574325 bytes in  75.735 seconds using pigz -c -9
 518170569 bytes in  75.898 seconds using pigz -c -9 -b 64
 518766300 bytes in  76.649 seconds using pigz -c -9 -b 512
 518699699 bytes in  77.513 seconds using pigz -c -9 -b 256
 517163600 bytes in  81.157 seconds using pigz -c -9 -b 32
 400795800 bytes in 104.774 seconds using xz -z -c -3 -T 0
 384321280 bytes in 130.551 seconds using xz -z -c -4 -T 0
 440812956 bytes in 138.224 seconds using xz -z -c -0
 408136360 bytes in 149.856 seconds using xz -z -c -0 -T 0 -e
 414591336 bytes in 167.587 seconds using xz -z -c -1
 389491264 bytes in 168.247 seconds using xz -z -c -1 -T 0 -e
 381135596 bytes in 178.825 seconds using xz -z -c -2 -T 0 -e
 373894068 bytes in 179.325 seconds using xz -z -c -3 -T 0 -e
 374142148 bytes in 179.662 seconds using xz -z -c -4 -T 0 -e
 366809628 bytes in 182.341 seconds using xz -z -c -5 -T 0 -e
 368988288 bytes in 183.542 seconds using xz -z -c -5 -T 0
 366947768 bytes in 192.392 seconds using xz -z -c -6 -T 0 -e
 359222592 bytes in 208.451 seconds using xz -z -c -7 -T 0 -e
 404333924 bytes in 209.236 seconds using xz -z -c -2
 367034284 bytes in 209.514 seconds using xz -z -c -6 -T 0
 359465936 bytes in 223.646 seconds using xz -z -c -7 -T 0
 355302128 bytes in 238.301 seconds using xz -z -c -8 -T 0
 355054912 bytes in 239.716 seconds using xz -z -c -8 -T 0 -e
 397590264 bytes in 259.024 seconds using xz -z -c -3
 380756200 bytes in 459.342 seconds using xz -z -c -4
 365909072 bytes in 587.287 seconds using xz -z -c -5
 403088120 bytes in 617.040 seconds using xz -z -c -0 -e
 363897956 bytes in 650.989 seconds using xz -z -c -6
 384136200 bytes in 669.630 seconds using xz -z -c -1 -e
 356567660 bytes in 673.724 seconds using xz -z -c -7
 352942048 bytes in 682.302 seconds using xz -z -c -8
 376549732 bytes in 695.215 seconds using xz -z -c -2 -e
 369914736 bytes in 706.825 seconds using xz -z -c -3 -e
 363587780 bytes in 732.890 seconds using xz -z -c -5 -e
 356411268 bytes in 734.169 seconds using xz -z -c -7 -e
 370125100 bytes in 740.674 seconds using xz -z -c -4 -e
 352687584 bytes in 764.315 seconds using xz -z -c -8 -e
 363778076 bytes in 767.257 seconds using xz -z -c -6 -e