#!/bin/sh

#html_dir=original_html
html_dir=new_breads
#bread_list=bread_url_list.txt
bread_list=new_breads.txt
cookie=

mkdir -p $html_dir


## Download each bread
sed s/" "/_/g $bread_list | sed s/_.*$// > html_list.tmp
wget --no-clobber --directory-prefix=$html_dir -i html_list.tmp
rm html_list.tmp

#generate image URL lists
rm -f urls.tmp
for line in $(find $html_dir/ -type f | grep "\.html"); do
    echo Processing $line
    cat $line | tr -d '\000' | sed s/">"/">\n"/g | grep file_store | grep 'src=\|href=' | sed s/'^.*src="'// | \
        sed s/'^.*href="'// | sed s/'"'.*$// | sed s/"\/player.php?v="// | sed s/"\&.*$"// |
        sed s/"^\/\/media.8kun.top"/"https:\/\/media.8kun.top"/g >> urls.tmp
done
# was needed for a while       | sed s/'\"\/\/media.8kun.top'/'\"https:\/\/media.8kun.top'/g
echo Sorting..
grep thumb < urls.tmp | sort -u > thumb_url.tmp
grep -v thumb < urls.tmp | sort -u > image_url.tmp
rm urls.tmp

mkdir -p questionable
for fn in $(find file_store/ -size +1050c -size -1070c); do
    echo "Removing $fn"
    mv $fn ./questionable/
done

#pull down new image files
mkdir -p file_store
mkdir -p file_store/thumb
wget --no-clobber --directory-prefix=./file_store -i image_url.tmp
wget --no-clobber --directory-prefix=./file_store/thumb -i thumb_url.tmp

rm image_url.tmp
rm thumb_url.tmp

##append new list to end of old list
cp bread_list.txt .bread_list.txt
cat .bread_list.txt new_breads.txt > bread_list.txt
echo "" > new_breads.txt

#move new htmls to the archive
mv new_breads/* original_html/

#remove bad files that may have been sent by the server (ddos midigation)
mkdir -p questionable
for fn in $(find file_store/ -size +1050c -size -1070c); do
    echo "Removing $fn"
    mv $fn ./questionable/
done