Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import aiohttp
- import asyncio
- import signal
- from asyncio import Event
- USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246"
- HEADERS = {'User-Agent': USER_AGENT}
- def warmup():
- # тут рутина, что добывает список с URL
- return []
- def cooldown(pending_urls):
- # тут рутина, чтоб схоронить необработанные URL после получения SIGINT
- pass
- # спиздил из https://medium.com/@rob.blackbourn/a-python-asyncio-cancellation-pattern-a808db861b84
- def make_cancellation_event(loop = None):
- # Create an event that gets set when the program is interrupted.
- cancellation_event = Event()
- if loop is None:
- loop = asyncio.get_event_loop()
- def cancel(name, num):
- msg = f'Received signal {name}'
- if num == signal.SIGINT:
- print(msg)
- else:
- print(msg)
- cancellation_event.set()
- for signame in ['SIGINT', 'SIGTERM']:
- signum = getattr(signal, signame)
- loop.add_signal_handler(signum, cancel, signame, signum)
- return cancellation_event
- async def get_file(session, url):
- async with session.get(url, headers=HEADERS) as resp:
- if resp.status == 200:
- content = await resp.read()
- with open(url[-7:]) as af:
- await af.write(content)
- return url
- async def main():
- untouched = warmup()
- cancellation_event = make_cancellation_event()
- cancellation_task = asyncio.create_task(cancellation_event.wait())
- async with aiohttp.ClientSession() as session:
- done, pending = await asyncio.wait(
- [cancellation_task] + [get_file(session, some) for some in untouched])
- # хуй понять, как тут дальше и когда вызвать cooldown(pending_urls)
- if __name__ == '__main__':
- asyncio.run(main())
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement