Advertisement
darkor

batch_get_items_soft

Sep 19th, 2023
459
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.15 KB | None | 0 0
  1. def batch_get_items_soft(keys: list, db_connection, table_name) -> list:
  2.     result = []
  3.     base_backoff_ms = 50
  4.     step = 100
  5.     current_attempt = 0
  6.     max_attempt_qty = 3
  7.     upper_limit = len(keys)
  8.  
  9.     for first_limit in range(0, upper_limit, step):
  10.         second_limit = first_limit + step
  11.         bunch_keys = keys[first_limit: second_limit]
  12.         last_item = second_limit >= upper_limit
  13.         last_attempt = current_attempt == max_attempt_qty
  14.  
  15.         try:
  16.             response = batch_get_item(bunch_keys, db_connection, table_name)
  17.         except ClientError as e:
  18.             raise db_exceptions.DbSelectException(str(e))
  19.  
  20.         result.extend(response)
  21.  
  22.         if last_attempt or last_item:
  23.             if last_attempt and not last_item:
  24.                 logger.error(f"Too many items for 'batch_get_item' {upper_limit}. Processed {second_limit}.")
  25.             break
  26.  
  27.         sleep_time_ms = base_backoff_ms * 2 ** current_attempt  # calculate exponential backoff
  28.         sleep_time_secs = sleep_time_ms / 1000  # convert milliseconds to seconds
  29.         current_attempt += 1
  30.         time.sleep(sleep_time_secs)
  31.  
  32.     return result
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement