Advertisement
Guest User

Untitled

a guest
Mar 21st, 2020
82
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.99 KB | None | 0 0
  1. import aiohttp
  2. import asyncio
  3. import signal
  4. from asyncio import Event
  5.  
  6. USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246"
  7. HEADERS = {'User-Agent': USER_AGENT}
  8.  
  9. def warmup():
  10.     # тут рутина, что добывает список с URL
  11.     return []
  12.  
  13. def cooldown(pending_urls):
  14.     # тут рутина, чтоб схоронить необработанные URL после получения SIGINT
  15.     pass
  16.  
  17. # спиздил из https://medium.com/@rob.blackbourn/a-python-asyncio-cancellation-pattern-a808db861b84
  18. def make_cancellation_event(loop = None):
  19.     # Create an event that gets set when the program is interrupted.
  20.     cancellation_event = Event()
  21.     if loop is None:
  22.         loop = asyncio.get_event_loop()
  23.     def cancel(name, num):
  24.         msg = f'Received signal {name}'
  25.         if num == signal.SIGINT:
  26.             print(msg)
  27.         else:
  28.             print(msg)
  29.         cancellation_event.set()
  30.     for signame in ['SIGINT', 'SIGTERM']:
  31.         signum = getattr(signal, signame)
  32.         loop.add_signal_handler(signum, cancel, signame, signum)
  33.     return cancellation_event
  34.  
  35. async def get_file(session, url):
  36.     async with session.get(url, headers=HEADERS) as resp:
  37.         if resp.status == 200:
  38.             content = await resp.read()
  39.             with open(url[-7:]) as af:
  40.                 await af.write(content)
  41.     return url
  42.  
  43. async def main():
  44.     untouched = warmup()
  45.     cancellation_event = make_cancellation_event()
  46.     cancellation_task = asyncio.create_task(cancellation_event.wait())
  47.     async with aiohttp.ClientSession() as session:
  48.         done, pending = await asyncio.wait(
  49.                 [cancellation_task] + [get_file(session, some) for some in untouched])
  50.        
  51.         # хуй понять, как тут дальше и когда вызвать cooldown(pending_urls)
  52.  
  53.  
  54. if __name__ == '__main__':
  55.     asyncio.run(main())
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement