Guest User

Untitled

a guest
Jul 9th, 2017
248
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.90 KB | None | 0 0
  1. import time
  2. import requests
  3. import threading
  4. from queue import Queue
  5.  
  6. q = Queue()
  7.  
  8. def worker():
  9. while True:
  10. url = q.get()
  11. print(url)
  12. data = url.split("/")
  13. title = data.pop()
  14. try:
  15. r = requests.get(url)
  16. status = r.status_code == 200
  17. if status:
  18. html = r.text
  19. file = open('downloaded' + title, 'w', encoding='utf8')
  20. file.write(html)
  21. file.close()
  22. except Exception as e:
  23. file = open('error.txt', 'a', encoding='utf8')
  24. file.write(url + "\n")
  25. file.close
  26. q.task_done()
  27.  
  28. for i in range(4):
  29. t = threading.Thread(target=worker)
  30. t.daemon = True
  31. t.start()
  32.  
  33. for item in open('urls.txt', 'r', encoding='utf8'):
  34. q.put(item.strip())
  35.  
  36. q.join()
  37.  
  38. print("Done all...")
Advertisement
Add Comment
Please, Sign In to add comment