daily pastebin goal
8%
SHARE
TWEET

Untitled

a guest Aug 13th, 2017 52 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. from tenacity import retry, wait_fixed, stop_after_attempt, before_log
  2. # ....
  3.  
  4. LAST_REQUESTS = {}
  5. LAST_REQUESTS_KEY = 0
  6.  
  7.  
  8. def httpreq(throttle_wait, **kwargs):
  9.     throttle_wait = throttle_wait
  10.     retry_kwargs = kwargs
  11.  
  12.     def real_decorator(function):
  13.         if len(retry_kwargs):
  14.  
  15.             @retry(**retry_kwargs)
  16.             async def wrapper(*args, **kwargs):
  17.                 if isinstance(LAST_REQUESTS_KEY, int):
  18.                     key = args[LAST_REQUESTS_KEY+1]
  19.                 else:
  20.                     key = kwargs[LAST_REQUESTS_KEY]
  21.                     kwargs.pop(LAST_REQUESTS_KEY)
  22.  
  23.                 date = LAST_REQUESTS.get(key, None)
  24.                 if not date:
  25.                     LAST_REQUESTS[key] = datetime.datetime.now()
  26.                 else:
  27.                     if (datetime.datetime.now() - date).total_seconds() < throttle_wait:
  28.                         await asyncio.sleep(throttle_wait)
  29.                     else:
  30.                         LAST_REQUESTS[key] = datetime.datetime.now()
  31.  
  32.                 return await function(*args, **kwargs)
  33.             return wrapper
  34.         else:
  35.             async def wrapper(*args, **kwargs):
  36.                 if isinstance(LAST_REQUESTS_KEY, int):
  37.                     key = args[LAST_REQUESTS_KEY + 1]
  38.                 else:
  39.                     key = kwargs[LAST_REQUESTS_KEY]
  40.                     kwargs.pop(LAST_REQUESTS_KEY)
  41.  
  42.                 date = LAST_REQUESTS.get(key, None)
  43.                 if not date:
  44.                     LAST_REQUESTS[key] = datetime.datetime.now()
  45.                 else:
  46.                     if (datetime.datetime.now() - date).total_seconds() < throttle_wait:
  47.                         await asyncio.sleep(throttle_wait)
  48.                     else:
  49.                         LAST_REQUESTS[key] = datetime.datetime.now()
  50.  
  51.                 return await function(*args, **kwargs)
  52.             return wrapper
  53.     return real_decorator
  54.  
  55. @httpreq(throttle_wait=settings.HTTP_REQUEST.get('THROTTLE_WAIT', None),
  56.          wait=wait_fixed(settings.HTTP_REQUEST.get('RETRY_WAIT_FIXED', None)),
  57.          stop=stop_after_attempt(settings.HTTP_REQUEST.get('RETRY_STOP_AFTER_ATTEMPT', None)))
  58. async def request(self, *args, **kwargs):
  59.     response = None
  60.     # ...
  61.     return response
RAW Paste Data
Top