Advertisement
Guest User

Async Bench: code with a queue

a guest
Oct 3rd, 2017
58
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.14 KB | None | 0 0
  1. import asyncio
  2. import aiohttp
  3.  
  4. LINK_FILE = open('links.txt', 'r')
  5. LINKS = LINK_FILE.readlines()
  6. TASKS_COUNT = 32
  7.  
  8.  
  9. async def download_task(queue):
  10.     while True:
  11.         img_url = await queue.get()
  12.  
  13.         async with aiohttp.ClientSession() as session:
  14.             async with session.get(img_url) as resp:
  15.                 while True:
  16.                     chunk = await resp.content.read(1024 * 64)
  17.                     if not chunk:
  18.                         break
  19.                     # writing chunk to file is unnecessary for a benchmark
  20.  
  21.         queue.task_done()
  22.         if queue.qsize() != 0:
  23.             print(f"\rImages left: {queue.qsize()}        ", end='')
  24.         else:
  25.             break
  26.  
  27.  
  28. async def main():
  29.     queue = asyncio.Queue()
  30.     for link in LINKS:
  31.         link = link.strip()
  32.         queue.put_nowait(link)
  33.  
  34.     consumers = []
  35.     for i in range(TASKS_COUNT):
  36.         consumer = asyncio.ensure_future(download_task(queue))
  37.         consumers.append(consumer)
  38.  
  39.     await queue.join()
  40.  
  41.  
  42. if __name__ == '__main__':
  43.     loop = asyncio.get_event_loop()
  44.     loop.run_until_complete(main())
  45.     loop.close()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement