Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import time
- import requests
- import threading
- from queue import Queue
- q = Queue()
- def worker():
- while True:
- url = q.get()
- print(url)
- data = url.split("/")
- title = data.pop()
- try:
- r = requests.get(url)
- status = r.status_code == 200
- if status:
- html = r.text
- file = open('downloaded' + title, 'w', encoding='utf8')
- file.write(html)
- file.close()
- except Exception as e:
- file = open('error.txt', 'a', encoding='utf8')
- file.write(url + "\n")
- file.close
- q.task_done()
- for i in range(4):
- t = threading.Thread(target=worker)
- t.daemon = True
- t.start()
- for item in open('urls.txt', 'r', encoding='utf8'):
- q.put(item.strip())
- q.join()
- print("Done all...")
Advertisement
Add Comment
Please, Sign In to add comment