packetstorm_getlists.sh

#!/bin/bash
# ~/development/wordlists/packetstorm_getlists.sh
#
#============================================================
#
#          FILE:  packetstorm_getlists.sh
#
#         USAGE:  ./packetstorm_getlists.sh
#
#   DESCRIPTION:  PacketStorm web crawler and password list extraction tool.
#
#       OPTIONS:  ---
#  REQUIREMENTS:  lynx
#          BUGS:  ---
#         NOTES:  lynx can be installed using the command:
#                     --- apt-get install lynx
#        AUTHOR:  tankd
#       COMPANY:  ---
#       VERSION:  1.0
#       CREATED:  01/25/12 02:42:08 PST
#      REVISION:  ---
#============================================================

for i in {1..8}; do
    var=$i
    for j in `lynx -dump http://packetstormsecurity.org/Crackers/wordlists/page${var}/ | grep -E --only-matching 'http\:.*' | grep 'download'`; do
        filename=$(echo ${j} | sed 's/http\:\/\/packetstormsecurity\.org\/files\/download\///g' | grep -E --only-matching '[a-z].*gz' | sed 's/[.]gz//g')
        wget -e robots=off $j
        if [ -f ${filename}.gz ]; then
            mv ${filename}.gz ${filename}.txt
        fi

        if [ -f ${filename}.zip ]; then
            mv ${filename}.zip ${filename}.txt
        fi
    done
done

# uncomment below to create one large wordlist.
#
#for i in `ls *.txt`; do
#    TITLE=$(echo $i | sed 's/.txt//g')
#   echo ***$TITLE*** >> huge_list.txt
#    cat $i >> huge_list.txt
#done