Advertisement
Guest User

Untitled

a guest
Jul 20th, 2019
152
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.90 KB | None | 0 0
  1. import os
  2. from dotenv import load_dotenv, find_dotenv
  3. import psycopg2
  4. import pandas as pd
  5. import boto3
  6.  
  7. class Redshift:
  8. def __init__(self):
  9. ''' Constructor for this class. '''
  10. # Create some members
  11. # find .env automagically by walking up directories until it's found
  12. dotenv_path = find_dotenv()
  13. # load up the entries as environment variables
  14. load_dotenv(dotenv_path)
  15. #credentials
  16. self.database_endpoint = os.environ.get("DATABASE_ENDPOINT")
  17. self.database_name = os.environ.get("DATABASE_NAME")
  18. self.database_user = os.environ.get("DATABASE_USER")
  19. self.database_password = os.environ.get("DATABASE_PASSWORD")
  20. self.port = os.environ.get("PORT")
  21.  
  22. def connect(self):
  23. #connect using psycopg2
  24. connection = psycopg2.connect(dbname=self.database_name, host=self.database_endpoint,
  25. port=self.port, user=self.database_user, password=self.database_password)
  26.  
  27. return connection
  28.  
  29.  
  30.  
  31. class S3:
  32. def __init__(self):
  33. ''' Constructor for this class. '''
  34. # Create some members
  35. # find .env automagically by walking up directories until it's found
  36. dotenv_path = find_dotenv()
  37. # load up t`he entries as environment variables
  38. load_dotenv(dotenv_path)
  39. #credentials
  40. self.AWS_KEY = os.environ.get("AWS_ACCESS_KEY")
  41. self.AWS_SECRET = os.environ.get("AWS_SECRET_ACCESS_KEY")
  42. self.AWS_BUCKET = os.environ.get("AWS_BUCKET")
  43.  
  44.  
  45.  
  46. def connect(self):
  47. client = boto3.client('s3', aws_access_key_id=self.AWS_ACCESS_KEY,aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY)
  48. return client
  49.  
  50. def read_file(self,file_name):
  51. client = self.connect()
  52. file_nm=client.get_object(Bucket=self.AWS_BUCKET, Key=file_name)
  53. df=pd.read_csv(file_nm['Body'],header=0)
  54. return df
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement