Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- from pyspark.sql import SQLContext
- from pyspark import SparkContext
- sc = SparkContext(appName="Connect Spark with Redshift")
- sql_context = SQLContext(sc)
- sc._jsc.hadoopConfiguration().set("fs.s3n.awsAccessKeyId", <ACCESSID>)
- sc._jsc.hadoopConfiguration().set("fs.s3n.awsSecretAccessKey", <ACCESSKEY>)
- df = sql_context.read
- .option("url", "jdbc:redshift://example.coyf2i236wts.eu-central- 1.redshift.amazonaws.com:5439/agcdb?user=user&password=pwd")
- .option("dbtable", "table_name")
- .option("tempdir", "bucket")
- .load()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement