Advertisement
Guest User

Untitled

a guest
Sep 22nd, 2017
84
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.27 KB | None | 0 0
  1. from pyspark.sql import SQLContext
  2. sqlContext = SQLContext(sc)
  3. df = sqlContext.load(source="jdbc", url="jdbc:postgresql://host/dbname", dbtable="schema.tablename")
  4.  
  5. os.environ['SPARK_CLASSPATH'] = "C:UsersACERNEW3DesktopSparkspark-1.3.0-bin-hadoop2.4postgresql-9.2-1002.jdbc3.jar"
  6.  
  7. pyspark --conf spark.executor.extraClassPath=<jdbc.jar> --driver-class-path <jdbc.jar> --jars <jdbc.jar> --master <master-URL>
  8.  
  9. from pyspark import SparkContext, SparkConf
  10. from pyspark.sql import DataFrameReader, SQLContext
  11. import os
  12.  
  13. sparkClassPath = os.getenv('SPARK_CLASSPATH', '/path/to/connector/postgresql-42.1.4.jar')
  14.  
  15. # Populate configuration
  16. conf = SparkConf()
  17. conf.setAppName('application')
  18. conf.set('spark.jars', 'file:%s' % sparkClassPath)
  19. conf.set('spark.executor.extraClassPath', sparkClassPath)
  20. conf.set('spark.driver.extraClassPath', sparkClassPath)
  21. # Uncomment line below and modify ip address if you need to use cluster on different IP address
  22. #conf.set('spark.master', 'spark://127.0.0.1:7077')
  23.  
  24. sc = SparkContext(conf=conf)
  25. sqlContext = SQLContext(sc)
  26.  
  27. url = 'postgresql://127.0.0.1:5432/postgresql'
  28. properties = {'user':'username', 'password':'password'}
  29.  
  30. df = DataFrameReader(sqlContext).jdbc(url='jdbc:%s' % url, table='tablename', properties=properties)
  31.  
  32. df.printSchema()
  33. df.show()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement