Guest User

smeg, now with autoresume

a guest
Feb 15th, 2013
82
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #!/bin/sh
  2. #
  3. # Sample use, runs 100 curls at a time, downloading range 1 to 2 million
  4. # for chunk in $(seq 100 199); do ./smeg $chunk &; done
  5. #
  6. # If restarted, automatically resumes where it left off using *.hostnames
  7. # output files.
  8.  
  9. chunk="$1"
  10. log=${chunk}0000.hostnames
  11. touch $log
  12. (for s in $(perl -lne 'BEGIN { $start=shift; $end=shift } chomp; s/\t.*//; $seen{$_}=1; END { for ($start..$end) { print $_ unless $seen{$_} } }' ${chunk}0000 ${chunk}9999 $log ); do
  13. echo $s >&2
  14. echo "ateam-site: $s"
  15. curl -s -H "X-Requested-With: XMLHttpRequest" -H "X-Xhrsource: posterous" -X GET -H "Cookie: _sharebymail_session_id=e55e807375f457efa9a22e091c0685c7; email=bugmenot%40trash-mail.com; _plogin=Veritas; logged_in_before=true" http://posterous.com/api/2/sites/$s
  16. done) | perl -ne '$|=1; if (/ateam-site: (\d+)/) { $site=$1 } if (/"full_hostname":"([^"]+)"/) { print "$site\t$1\n"; }' >> $log
RAW Paste Data