Guest User

Untitled

a guest
Dec 6th, 2017
94
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.48 KB | None | 0 0
  1. # script with 2 ways to push to postgresql database
  2. import pandas as pd
  3. import psycopg2
  4. import logging
  5. import configparser
  6. from sqlalchemy import create_engine
  7.  
  8. # config parser
  9. config = configparser.RawConfigParser()
  10. config.read('auth.conf')
  11.  
  12. # read from auth.conf file
  13. OBJECTSTORE_PASSWORD = config['postgresql_dataservices']['PS_PASSWORD']
  14.  
  15. PS_ENGINE = {'dialect': config.get('postgresql_dataservices','PS_DIALECT'),
  16. 'user': config.get('postgresql_dataservices','PS_USER'),
  17. 'password': OBJECTSTORE_PASSWORD,
  18. 'host': config.get('postgresql_dataservices','PS_HOST'),
  19. 'port': config.get('postgresql_dataservices','PS_PORT'),
  20. 'database': config.get('postgresql_dataservices', 'PS_DATABASE'),
  21. 'schema' : config.get('postgresql_dataservices', 'PS_SCHEMA')
  22. }
  23.  
  24. # db_url to be put in to_sql
  25. db_url = ('{dialect}://{user}:{password}@{host}:{port}/{database}'
  26. .format(dialect=PS_ENGINE['dialect'],
  27. user=PS_ENGINE['user'],
  28. password=PS_ENGINE['password'],
  29. host=PS_ENGINE['host'],
  30. port=PS_ENGINE['port'],
  31. database=PS_ENGINE['database']))
  32.  
  33. engine = create_engine(db_url)
  34.  
  35. ## df.to_sql(OUTPUT_FILE, con=engine, schema='passagiersvaart', if_exists='replace')
  36.  
  37. ## write to postgresql with psycopg2 module
  38. psycopg2_connect = (psycopg2.connect("dbname={} user={} host={} port ={} password={} sslmode = {}"
  39. .format(PS_ENGINE['database'],
  40. PS_ENGINE['user'],
  41. PS_ENGINE['host'],
  42. PS_ENGINE['port'],
  43. PS_ENGINE['password'],
  44. 'disable')))
  45.  
  46. def query_data_postgresql(sql):
  47. """input: sql statement
  48. output: Pandas dataframe"""
  49. print("loading data ...")
  50.  
  51. column_names = []
  52. data_rows = []
  53. with psycopg2_connect as conn:
  54. with conn.cursor() as cursor:
  55. cursor.execute(sql)
  56. column_names = [desc[0] for desc in cursor.description]
  57. for row in cursor:
  58. data_rows.append(row)
  59. df = pd.DataFrame(data_rows, columns=column_names)
  60.  
  61. return df
Add Comment
Please, Sign In to add comment