from requests_futures.sessions import FuturesSession
import os
urls = []
num_workers = os.cpu_count()
session = FuturesSession(max_workers=num_workers)
while urls:
    futures = []
    for i in range(num_workers):
        if not urls:
            break
        # submit a request
        futures.append(session.get(urls.pop(0)))
    # num_workers jobs have been submitted and are being worked on in the background
    # wait for each of them to finish
    for future in futures:
        resp = future.result()
        print(resp.status_code)
        # do something with the response
    # all of this batch has completed now, go around the while loop and start another