Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- url_func_dict = {}
- for urls in promos_urls:
- for url in urls:
- url_func_dict[url] = promos_urls[urls]
- with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
- # Start the load operations and mark each future with its URL
- future_to_url = {executor.submit(get_html, u): u for u in url_func_dict}
- for future in concurrent.futures.as_completed(future_to_url):
- url = future_to_url[future]
- html = future.result()
- if html[0] == 'error':
- error_counter = error_counter + 1
- print(html[1])
- continue
- else:
- i=url_func_dict[url](html, rooms, url)
- scraped_promos = scraped_promos+i
- print(str(len(i))+' promos were scraped from '+url)
- if not prev_time:
- prev_time = start_time
- print("url took", time.time() - prev_time, "sec to run")
- prev_time = time.time()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement