Guest User

Untitled

a guest
Oct 21st, 2017
106
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.53 KB | None | 0 0
  1. import pandas as pd
  2. from sqlalchemy import create_engine
  3. from datetime import date
  4. from s3fs import S3FileSystem
  5.  
  6.  
  7. def create_db_engine(database=None,user=None,password=None,host=None,port=5432):
  8. """
  9. Create sqlalchemy postgres engine from settings
  10. """
  11. engine_string = "postgresql+psycopg2://{user}:{password}@{host}:{port}/{database}".format(
  12. user = user,
  13. password = password,
  14. host = host,
  15. port = port,
  16. database = database,
  17. )
  18. return create_engine(engine_string)
  19.  
  20. def create_db_engine_from_settings_dict(database_settings):
  21. """
  22. Create sqlalchemy postgres engine from django db settings dict
  23. """
  24. return create_db_engine(
  25. database = database_settings['NAME'],
  26. user = database_settings['USER'],
  27. password = database_settings['PASSWORD'],
  28. host = database_settings['HOST'],
  29. )
  30.  
  31. def write_df_to_s3(df, s3_bucket, s3_key, encoding='utf-8', index=False, **kwargs):
  32. """
  33. Writes a pandas dataframe to a file on AWS S3
  34. Utilizes s3fs pandas integration
  35. Passes some alternate kwarg defaults for DataFrame.to_csv()
  36.  
  37. Arguments:
  38. df (pd.DataFrame): dataframe to write
  39. s3_bucket (str): s3 bucket name, e.g. "my-bucket"
  40. s3_key (str): s3 bucket key to write file as, e.g. "data/myfile.csv"
  41. kwargs: keyword arguments to pass to pd.DataFrame.to_csv()
  42. """
  43. s3 = S3FileSystem(anon=False)
  44. s3file = s3.open("{}/{}".format(s3_bucket, s3_key), mode='wb')
  45. df.to_csv(s3file, encoding=encoding, index=index, **kwargs)
Add Comment
Please, Sign In to add comment