adfasdfadsfasdf

Untitled

May 10th, 2023
137
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 2.00 KB | None | 0 0
  1. ```py
  2. def collect_chunks(token_list: list, start: list, end: list, include_end: bool = True):
  3.     output = []
  4.    
  5.     # This is the start index where we look for first index such that [idx + len(start)] --> start
  6.     start_idx = 0
  7.    
  8.     while start_idx < len(token_list):
  9.        
  10.         # end must be after the start, so set it to start + 1
  11.         end_idx = start_idx + 1
  12.  
  13.         # Verify that there is still space in the token list in the first condition.
  14.         # In the second condition, check that start that was passed in ==
  15.         # token_list starting at start_idx till start_idx + len(start).
  16.         # Recollect that start_idx is just an idx, and start_idx + len(start) is a
  17.         # subsequence of token_list of size len(start)
  18.         if (start_idx + len(start) <= len(token_list)) and (start == token_list[start_idx:start_idx + len(start)]):
  19.            
  20.             # Search all possible end idxs from the end of the start to the end of the string.
  21.             # Give padding for the size of the end token.
  22.             for possible_end_idx in range(end_idx + len(start) - 1, len(token_list) - len(end) + 1):
  23.  
  24.                 # Similar to the first condition, check if from end idx to len(end) == passed in end.
  25.                 if end == token_list[possible_end_idx: possible_end_idx + len(end)]:
  26.                     # Just include the whole end if that param is true.
  27.                     end_idx = possible_end_idx + len(end) if include_end else possible_end_idx
  28.                 break
  29.  
  30.         # Self-explanatory, join everything from start to end. If we never found anything this is
  31.         # start:start+1 because we set end_idx to start + 1 at the beginning.
  32.         # This adds just the character if we found nothing.
  33.         output.append("".join(token_list[start_idx:end_idx]))
  34.  
  35.         assert end_idx > start_idx
  36.        
  37.         # We searched everything from start to end, so no need to research. set start to old end.
  38.         start_idx = end_idx
  39.        
  40.     return output
  41. ```
Advertisement
Add Comment
Please, Sign In to add comment