Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def iterator2dataframes(iterator, chunk_size: int):
- """Turn an iterator into multiple small pandas.DataFrame
- This is a balance between memory and efficiency"""
- records = []
- frames = []
- for i, record in enumerate(iterator):
- records.append(record)
- if i % chunk_size == chunk_size - 1:
- frames.append(pd.DataFrame(records))
- records = []
- if records:
- frames.append(pd.DataFrame(records))
- return pd.concat(frames)
- df = iterator2dataframes(cursor, 25000)
Add Comment
Please, Sign In to add comment