daily pastebin goal
47%
SHARE
TWEET

Untitled

a guest Aug 13th, 2017 54 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. from tenacity import retry, wait_fixed, stop_after_attempt, before_log
  2. # ....
  3.  
  4. LAST_REQUESTS = {}
  5. LAST_REQUESTS_KEY = 0
  6.  
  7.  
  8. def httpreq(throttle_wait, **kwargs):
  9.     throttle_wait = throttle_wait
  10.     retry_kwargs = kwargs
  11.  
  12.     def real_decorator(function):
  13.         if len(retry_kwargs):
  14.  
  15.             @retry(**retry_kwargs)
  16.             async def wrapper(*args, **kwargs):
  17.                 if isinstance(LAST_REQUESTS_KEY, int):
  18.                     key = args[LAST_REQUESTS_KEY+1]
  19.                 else:
  20.                     key = kwargs[LAST_REQUESTS_KEY]
  21.                     kwargs.pop(LAST_REQUESTS_KEY)
  22.  
  23.                 date = LAST_REQUESTS.get(key, None)
  24.                 if not date:
  25.                     LAST_REQUESTS[key] = datetime.datetime.now()
  26.                 else:
  27.                     if (datetime.datetime.now() - date).total_seconds() < throttle_wait:
  28.                         await asyncio.sleep(throttle_wait)
  29.                     else:
  30.                         LAST_REQUESTS[key] = datetime.datetime.now()
  31.  
  32.                 return await function(*args, **kwargs)
  33.             return wrapper
  34.         else:
  35.             async def wrapper(*args, **kwargs):
  36.                 if isinstance(LAST_REQUESTS_KEY, int):
  37.                     key = args[LAST_REQUESTS_KEY + 1]
  38.                 else:
  39.                     key = kwargs[LAST_REQUESTS_KEY]
  40.                     kwargs.pop(LAST_REQUESTS_KEY)
  41.  
  42.                 date = LAST_REQUESTS.get(key, None)
  43.                 if not date:
  44.                     LAST_REQUESTS[key] = datetime.datetime.now()
  45.                 else:
  46.                     if (datetime.datetime.now() - date).total_seconds() < throttle_wait:
  47.                         await asyncio.sleep(throttle_wait)
  48.                     else:
  49.                         LAST_REQUESTS[key] = datetime.datetime.now()
  50.  
  51.                 return await function(*args, **kwargs)
  52.             return wrapper
  53.     return real_decorator
  54.  
  55. @httpreq(throttle_wait=settings.HTTP_REQUEST.get('THROTTLE_WAIT', None),
  56.          wait=wait_fixed(settings.HTTP_REQUEST.get('RETRY_WAIT_FIXED', None)),
  57.          stop=stop_after_attempt(settings.HTTP_REQUEST.get('RETRY_STOP_AFTER_ATTEMPT', None)))
  58. async def request(self, *args, **kwargs):
  59.     response = None
  60.     # ...
  61.     return response
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top