Advertisement
Guest User

Untitled

a guest
Jun 25th, 2019
87
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.70 KB | None | 0 0
  1. s3 = boto3.client('s3')
  2. bucket = bucket_name
  3.  
  4. file_name = file_key
  5.  
  6. sql_stmt = """SELECT S.* FROM s3object S LIMIT 10"""
  7.  
  8.  
  9. req = s3.select_object_content(
  10. Bucket=bucket_name,
  11. Key=file_key,
  12. ExpressionType='SQL',
  13. Expression=sql_stmt,
  14. InputSerialization = {'CSV': {'FileHeaderInfo': 'USE'}},
  15. OutputSerialization = {'CSV': {}},
  16. )
  17.  
  18. records = []
  19. for event in req['Payload']:
  20. if 'Records' in event:
  21. records.append(event['Records']['Payload'])
  22. elif 'Stats' in event:
  23. stats = event['Stats']['Details']
  24.  
  25.  
  26. file_str = ''.join(r.decode('utf-8') for r in records)
  27.  
  28. select_df = pd.read_csv(StringIO(file_str))
  29. print(len(select_df))
  30. df = pd.DataFrame(select_df)
  31. print(df)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement