export JAVA_HOME=/usr/lib/jvm/jdk1.7.0_51
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# get crawl_path and solr_url from arguments, set to default values if not given
CRAWL_PATH="$1"
SOLR_URL="$2"
if [ "$CRAWL_PATH" == "" ]; then
   CRAWL_PATH="crawl"
fi
if [ "$SOLR_URL" == "" ]; then
   SOLR_URL="http://localhost:8983/solr"
fi

# rename seed.txt to seed_timestamp.txt
timestamp=$(date +"%s")
OLD_DIR=$DIR"/urls1/seed.txt"
SEED_DIR=$DIR"/urls1/seed_"$timestamp".txt"
mv $OLD_DIR $SEED_DIR

# re-create the seed.txt if not exist; use this file to accept new records
echo '' >> $OLD_DIR

# generate segments from the urls in seed_timestamp.txt
$DIR/bin/nutch freegen $SEED_DIR $DIR/$CRAWL_PATH/segments
# find the generated segments
SEGMENT=`ls $DIR/$CRAWL_PATH/segments/ | sort -n | tail -n 1`

# fetch the segment
$DIR/bin/nutch fetch $DIR/$CRAWL_PATH/segments/$SEGMENT

# parse the segment
$DIR/bin/nutch parse $DIR/$CRAWL_PATH/segments/$SEGMENT

# updatedb with this segment
$DIR/bin/nutch updatedb $DIR/$CRAWL_PATH/crawldb $DIR/$CRAWL_PATH/segments/$SEGMENT

# link inversion
$DIR/bin/nutch invertlinks $DIR/$CRAWL_PATH/linkdb $DIR/$CRAWL_PATH/segments/$SEGMENT

# index with solr
$DIR/bin/nutch solrindex $SOLR_URL $DIR/$CRAWL_PATH/crawldb -linkdb $DIR/$CRAWL_PATH/linkdb $DIR/$CRAWL_PATH/segments/$SEGMENT