Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from io import StringIO
- import pandas as pd
- csv_buffer = StringIO()
- df.to_csv(csv_buffer, chunksize=1000)
- s3_resource = boto3.resource('s3')
- s3_resource.Object(bucket, 'df.csv').put(Body=csv_buffer.getvalue())
- >>> from io import StringIO
- ... from itertools import islice
- ... import sys
- ...
- ... import numpy as np
- ... import pandas as pd
- ...
- ... df = pd.DataFrame(np.arange(300).reshape(100, -1))
- ... csv_buffer = StringIO()
- ... df.to_csv(csv_buffer)
- ... csv_buffer.seek(0)
- ...
- ... # Account for indivisibility (scoop up a remainder on the final slice).
- ... chunksize = 33
- ... rowsize = df.shape[1]
- ... slices = [(0, chunksize)] * (rowsize - 1) + [(0, sys.maxsize)]
- ... chunks = (tuple(islice(csv_buffer, i, j)) for i, j in slices)
- ...
- >>> next(chunks)
- (',0,1,2n',
- '0,0,1,2n',
- '1,3,4,5n',
- '2,6,7,8n',
- '3,9,10,11n',
- '4,12,13,14n',
- '5,15,16,17n',
- '6,18,19,20n',
- '7,21,22,23n',
- '8,24,25,26n',
- '9,27,28,29n',
- '10,30,31,32n',
- '11,33,34,35n',
- '12,36,37,38n',
- '13,39,40,41n',
- '14,42,43,44n',
- '15,45,46,47n',
- '16,48,49,50n',
- '17,51,52,53n',
- '18,54,55,56n',
- '19,57,58,59n',
- '20,60,61,62n',
- '21,63,64,65n',
- '22,66,67,68n',
- '23,69,70,71n',
- '24,72,73,74n',
- '25,75,76,77n',
- '26,78,79,80n',
- '27,81,82,83n',
- '28,84,85,86n',
- '29,87,88,89n',
- '30,90,91,92n',
- '31,93,94,95n')
Add Comment
Please, Sign In to add comment