merkator

not ready

Nov 22nd, 2011
110
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.29 KB | None | 0 0
  1. #-*-coding: cp1251 -*-
  2. import urllib2
  3. import cookielib
  4.  
  5. # time module for performance metrics
  6. import time
  7. # re module for simple replace
  8. import re
  9. # error handler for 404/401/403 and etc
  10. from urllib2 import HTTPError
  11. # let's try multithreading to speed up our app
  12. import thread
  13.  
  14. start = time.time()
  15. inp = open("input.txt", "r")
  16.  
  17. st = inp.read().split("\n")
  18.  
  19. cookie = cookielib.CookieJar()  
  20. opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
  21. urllib2.install_opener(opener)
  22.  
  23. fn = st
  24.  
  25. for i in xrange(len(st)):
  26.     fn[i] = re.sub("[\?=\/]","_",fn[i][7:])
  27.  
  28. lst = [st[0::2], st[1::2]]
  29. fnlist = [fn[0::2], fn[1::2]]
  30. l = [len(lst[0]), len(lst[1])]
  31. c = [0, 0]
  32.  
  33. def f1(threadnum):
  34.     while(c[threadnum]<l[threadnum]):
  35.         #assert threadnum>=0 and threadnum<=1
  36.         a = lst[threadnum][c[threadnum]]
  37.         fname = fnlist[threadnum][c[threadnum]]
  38.         c[threadnum]+=1
  39.         try:
  40.             out=open("".join(fname[7:])+".txt", "w")
  41.             site = urllib2.urlopen(a)
  42.             print(site.read())
  43.             out.close()
  44.         except HTTPError, e:
  45.             print e.code
  46.  
  47. try:
  48.     thread.start_new_thread( f1, (0, ) )
  49.     thread.start_new_thread( f1, (1, ) )
  50. except Exception, e:
  51.     print e
  52.     print "Failed to create new thread"
  53.  
  54. end = time.time()
  55. print "Elapsed Time: %s" % (end - start)
Advertisement
Add Comment
Please, Sign In to add comment