Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import asyncio
- import aiohttp
- LINK_FILE = open('links.txt', 'r')
- LINKS = LINK_FILE.readlines()
- TASKS_COUNT = 32
- async def download_task(queue):
- while True:
- img_url = await queue.get()
- async with aiohttp.ClientSession() as session:
- async with session.get(img_url) as resp:
- while True:
- chunk = await resp.content.read(1024 * 64)
- if not chunk:
- break
- # writing chunk to file is unnecessary for a benchmark
- queue.task_done()
- if queue.qsize() != 0:
- print(f"\rImages left: {queue.qsize()} ", end='')
- else:
- break
- async def main():
- queue = asyncio.Queue()
- for link in LINKS:
- link = link.strip()
- queue.put_nowait(link)
- consumers = []
- for i in range(TASKS_COUNT):
- consumer = asyncio.ensure_future(download_task(queue))
- consumers.append(consumer)
- await queue.join()
- if __name__ == '__main__':
- loop = asyncio.get_event_loop()
- loop.run_until_complete(main())
- loop.close()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement