Guest User

Untitled

a guest
Jun 8th, 2018
103
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.61 KB | None | 0 0
  1. import org.apache.spark.sql._
  2. import com.databricks._
  3.  
  4. val sqlc = SparkSession.builder.appName("test_job").getOrCreate()
  5. sqlc.read
  6. .format("com.databricks.spark.redshift")
  7. .option("url", "jdbc:postgresql://{servername}.redshift.amazonaws.com:{portnumber}/prod")
  8. .option("user", "username")
  9. .option("password", "password")
  10. .option("query", "select train_nbr from datamart.train_ticket")
  11. .option("forward_spark_s3_credentials", "true")
  12. .option("tempdir", "s3n://{path}}/tempfile")
  13. .load().createOrReplaceTempView("temp")
  14.  
  15. val allrecords = sqlc.sql("select count(1) from temp")
  16. allrecords.show()
Add Comment
Please, Sign In to add comment