tlist.sh

#!/bin/bash
#############################################################
# This script searches twitter for a given search term,
# and parses the results into a proper wordlist thing.
# USAGE: ./tlist.sh {keyword}
# EXAMPLE: counterculture.txt, http://pastebin.com/XuCwMJW6
#          cyberculture.txt, http://pastebin.com/Le2G44yf
# EXTERNALS: stop-words.txt, http://pastebin.com/ihkVeWeX
#############################################################

if [ -z "$1" ];
then
    echo -e "\n::::::\n\ntlist creates a list of words, one word per line.\nUse a keyword as a seed to your wordlist.\n\nUSAGE :: tlist {search-term}\n\n::::::\n"
    exit 0
fi

### THE MEAT:: ###
curl -s "http://search.twitter.com/search.json?q=$1&rpp=500" |  # Ask Twitter for words!
tr "," \\n |        # Parse JSON into 1 key-value pair per line.
grep "^\"text" |    # Extract tweet text only.
cut -d"\"" -f4- |   # Remove JSON Key, from each line.
tr " " \\n |        # Put every word on it's own line.
sed s/\"//g |       # Remove all double quotes.
sed s/\^\#//g |     # Remove hashtags.
sed s/\^\@//g |     # Remove mentions text.
grep -v "^http:" |  # Don't include url's.
grep -v "\\\\" |        # Remove back slashes.
tr -cs A-Za-z '\n' |
tr A-Z a-z |
egrep -v -f /usr/local/lib/stop-words.txt