Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def batch_get_items_soft(keys: list, db_connection, table_name) -> list:
- result = []
- base_backoff_ms = 50
- step = 100
- current_attempt = 0
- max_attempt_qty = 3
- upper_limit = len(keys)
- for first_limit in range(0, upper_limit, step):
- second_limit = first_limit + step
- bunch_keys = keys[first_limit: second_limit]
- last_item = second_limit >= upper_limit
- last_attempt = current_attempt == max_attempt_qty
- try:
- response = batch_get_item(bunch_keys, db_connection, table_name)
- except ClientError as e:
- raise db_exceptions.DbSelectException(str(e))
- result.extend(response)
- if last_attempt or last_item:
- if last_attempt and not last_item:
- logger.error(f"Too many items for 'batch_get_item' {upper_limit}. Processed {second_limit}.")
- break
- sleep_time_ms = base_backoff_ms * 2 ** current_attempt # calculate exponential backoff
- sleep_time_secs = sleep_time_ms / 1000 # convert milliseconds to seconds
- current_attempt += 1
- time.sleep(sleep_time_secs)
- return result
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement