Advertisement
Guest User

Untitled

a guest
Dec 7th, 2016
84
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.01 KB | None | 0 0
  1.     url_func_dict = {}
  2.     for urls in promos_urls:
  3.         for url in urls:
  4.             url_func_dict[url] = promos_urls[urls]
  5.     with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
  6.         # Start the load operations and mark each future with its URL
  7.         future_to_url = {executor.submit(get_html, u): u for u in url_func_dict}
  8.         for future in concurrent.futures.as_completed(future_to_url):
  9.             url = future_to_url[future]
  10.             html = future.result()
  11.             if html[0] == 'error':
  12.                 error_counter = error_counter + 1
  13.                 print(html[1])
  14.                 continue
  15.             else:      
  16.                 i=url_func_dict[url](html, rooms, url)
  17.                 scraped_promos = scraped_promos+i
  18.                 print(str(len(i))+' promos were scraped from '+url)
  19.             if not prev_time:
  20.                 prev_time = start_time
  21.             print("url took", time.time() - prev_time, "sec to run")
  22.             prev_time = time.time()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement