Guest User

Untitled

a guest
Mar 19th, 2018
97
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.27 KB | None | 0 0
  1. from io import StringIO
  2.  
  3. import pandas as pd
  4.  
  5. csv_buffer = StringIO()
  6. df.to_csv(csv_buffer, chunksize=1000)
  7. s3_resource = boto3.resource('s3')
  8. s3_resource.Object(bucket, 'df.csv').put(Body=csv_buffer.getvalue())
  9.  
  10. >>> from io import StringIO
  11. ... from itertools import islice
  12. ... import sys
  13. ...
  14. ... import numpy as np
  15. ... import pandas as pd
  16. ...
  17. ... df = pd.DataFrame(np.arange(300).reshape(100, -1))
  18. ... csv_buffer = StringIO()
  19. ... df.to_csv(csv_buffer)
  20. ... csv_buffer.seek(0)
  21. ...
  22. ... # Account for indivisibility (scoop up a remainder on the final slice).
  23. ... chunksize = 33
  24. ... rowsize = df.shape[1]
  25. ... slices = [(0, chunksize)] * (rowsize - 1) + [(0, sys.maxsize)]
  26. ... chunks = (tuple(islice(csv_buffer, i, j)) for i, j in slices)
  27. ...
  28.  
  29. >>> next(chunks)
  30. (',0,1,2n',
  31. '0,0,1,2n',
  32. '1,3,4,5n',
  33. '2,6,7,8n',
  34. '3,9,10,11n',
  35. '4,12,13,14n',
  36. '5,15,16,17n',
  37. '6,18,19,20n',
  38. '7,21,22,23n',
  39. '8,24,25,26n',
  40. '9,27,28,29n',
  41. '10,30,31,32n',
  42. '11,33,34,35n',
  43. '12,36,37,38n',
  44. '13,39,40,41n',
  45. '14,42,43,44n',
  46. '15,45,46,47n',
  47. '16,48,49,50n',
  48. '17,51,52,53n',
  49. '18,54,55,56n',
  50. '19,57,58,59n',
  51. '20,60,61,62n',
  52. '21,63,64,65n',
  53. '22,66,67,68n',
  54. '23,69,70,71n',
  55. '24,72,73,74n',
  56. '25,75,76,77n',
  57. '26,78,79,80n',
  58. '27,81,82,83n',
  59. '28,84,85,86n',
  60. '29,87,88,89n',
  61. '30,90,91,92n',
  62. '31,93,94,95n')
Add Comment
Please, Sign In to add comment