###
#
# full script contains a locking mechanism, you should implement that if you want to use it
#
###

#!/bin/bash
# ------------------------------------------------------------------------------
# File:    wget_cdn_logs
# Descrip: transfers all files from the ftp which are not in a file which
#          contains a list of already transfered/processed files so that we
#          are able to delete files after processing (to avoid reprocessing)
#
# Usage:   wget_cdn_logs
#
# Project: XXX
#
# Update:  2011-08-05 TS-MMS/joko: (initial)
# KnownErrors/ToDos:
#
# ------------------------------------------------------------------------------


WORKING_DIR="/store/urchin/logs/cdn"
DOWNLOADED_FILELIST="${WORKING_DIR}/.files_done"
DOWNLOAD_FILELIST="${WORKING_DIR}/.downloadlist"

# this is also hardcoded down in a sed because of the nice /...
FTP_ADDR="ftp://upload.xxx.de/"
FTP_USER="xxx-logfile"
FTP_PASS="xxx"


###
#
# Main Code
#
###

cd ${WORKING_DIR} 2>/dev/null
if [ "$?" != "0" ]; then echo "working dir does not exist" 1>&2; exit 1; fi

# remove the wget ftp listing if there is one, some wget versions have a bug
# which always appends to existing .listing files
rm .listing 2>/dev/null

# now download only the listing (by rejecting every real file), we will use
# this to determine what we haven't already downloaded from the ftp
wget --quiet --no-remove-listing --recursive --level=0 --no-directories --ftp-user=${FTP_USER} --ftp-password=${FTP_PASS} --reject "*" "${FTP_ADDR}*"

# now we edit the .listing file to our needs, we only need the last column
# and we want to prepend everything with the FTP_ADDR
## we could do this inline with --i but the second wget will overwrite our
## .listing which we still need as the new list of already dled files
sed -n 's/.* /ftp:\/\/upload.xxx.de\//;p' .listing > .listing_transformed

# now calculate the diff of the listing and of the file list of dled files the
# output is what needs to be download
comm -23 .listing_transformed ${DOWNLOADED_FILELIST} > ${DOWNLOAD_FILELIST}

# now wget everything that is in the downloadlist
wget --quiet --recursive --level=0 --continue --no-directories --ftp-user=${FTP_USER} --ftp-password=${FTP_PASS} --input-file=${WORKING_DIR}/.downloadlist

# if the download was successful then update list of downloaded files
if [ "$?" -eq "0" ]; then
        cp .listing_transformed ${DOWNLOADED_FILELIST}
fi

# now cleanup some stuff
rm .listing_transformed .listing .downloadlist 2>/dev/null

exit 0