get_modarchive_mods.sh

#!/bin/bash

######################################################################################
get_modules() {
    local nr=$1
    local dir="../Downloaded"

    [ ! -d $dir ] && mkdir $dir

    local    URL="http://modarchive.org/index.php?"
    local   site="${URL}request=view_top_favourites&page=${nr}#mods"
    local SEARCH="${URL}request=search&submit=Find&search_type=filename_or_songtitle&query="
    local  GENRE="${URL}request=view_by_moduleid&query="

    echo "Page: $nr"

    [ ! -f downloaded ] && touch downloaded
    [ ! -f deleted ] && touch deleted

    for i in `lynx -dump -listonly "$site" | grep api | grep downloads | rev | cut -d" " -f1 | rev | sort | uniq` ; do

        name=$(echo $i | cut -d# -f2)
        have=$(cat downloaded deleted | grep -ci "$name")

        if [[ $have -eq 0 ]]; then
            echo Getting: \"$name\"
            wget --quiet -O "$dir/$name" "$i"

            if [ -s "$dir/$name" ]; then

                # get the module id by searching it by file name
                if [[ $( lynx -dump -listonly ${SEARCH}$name | grep api | wc -l) -gt 1 ]]; then
                    id=UNKN
                    genre=n/a
                else
                    id=$(lynx -dump -listonly "${SEARCH}$name" | grep api)
                    id=${id#*=}
                    id=${id%\#*}

                    # get the genre of the module searching by ID
                    genre=$(lynx -dump -nolist "${GENRE}$id" | grep "Genre:" | head -1)
                    genre=${genre#*: }
                fi

                echo "$id;$name;$genre" >> downloaded
            fi
        fi
    done

    7zpack_sep.sh $dir
}

######################################################################################
if [ -n "$1" ]; then

    [ -n "$1" ] && start=1    end=$1 && shift
    [ -n "$1" ] && start=$end end=$1

    for (( j = $start; j <= $end ; j++ )); do
        get_modules $j
    done

else

    if [ -s last_page ]; then
        next=$(( $( cat last_page ) + 1 ))
    else
        echo 1 > last_page
        next=1
    fi

    if [ -v next ]; then
        get_modules $next
        echo $next > last_page
    fi
fi