Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import pandas as pd
- from tqdm import tqdm
- from alive_progress import alive_bar
- import time
- from pathlib import Path
- import os
- def chunker(seq, size):
- # from http://stackoverflow.com/a/434328
- return (seq[pos:pos + size] for pos in range(0, len(seq), size))
- def progress_1(df, filename):
- chunksize = int(len(df) / 100) if len(df) >=100000 else len(df)
- print('chunksize: ', chunksize)
- if os.path.isfile(filename):
- os.remove(filename)
- with tqdm(total=len(df)) as pbar:
- for i, cdf in enumerate(chunker(df, chunksize)):
- mode = "w" if i == 0 else "a"
- cdf.to_csv(filename + '.csv', index=False, header=False, mode=mode)
- pbar.update(chunksize)
- def progress_2(df, filename):
- chunksize = int(len(df) / 100) if len(df) >=100000 else len(df)
- items = range(0, len(df), chunksize)
- if os.path.isfile(filename):
- os.remove(filename)
- with alive_bar(total=len(items), bar='bubbles', spinner='dots_reverse') as bar:
- for item, item2 in zip(items, enumerate(chunker(df, chunksize))):
- mode = "w" if item2[0] == 0 else "a"
- item2[1].to_csv(filename + '.csv', index=False, header=False, mode=mode)
- bar()
- df = pd.DataFrame({'a': range(0, 100000)})
- # drive_path = str(Path(os.path.abspath(os.path.dirname(__file__))))
- drive_path = 'E:'
- progress_1(df, drive_path + '/out_f.csv')
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement