#!/bin/bash
# download files using wget
# URLs go in queue.txt

cat queue.txt >> download.log

while [ -s queue.txt ]; do
    wget -r --no-parent --reject "index.html*" -e robots=off $(head -n 1 queue.txt) | tee download.log
    sed -i '1d' queue.txt
done