Advertisement
Guest User

Untitled

a guest
Feb 4th, 2017
158
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.75 KB | None | 0 0
  1. val sparkSession = SparkSession
  2. .builder
  3. .master("local")
  4. .appName("CollabrativeFilter")
  5. .config("spark.sql.warehouse.dir", "file:///c:/Temp/spark-warehouse")
  6. .config("spark.sql.crossJoin.enabled", true)
  7. .getOrCreate()
  8.  
  9. val awsAccessKeyId = "value"
  10. val awsSecretAccessKey = "value"
  11. val redshiftDBName = "value"
  12. val redshiftUserId = "value"
  13. val redshiftPassword = "value"
  14. val redshifturl = "value"
  15. val jdbcURL = s"jdbc:redshift://$redshifturl/$redshiftDBName?user=$redshiftUserId&password=$redshiftPassword"
  16.  
  17. val tempS3Dir = "s3n:accessid:secretkey@bucket/"
  18.  
  19.  
  20. val eventsDF = sparkSession.read
  21. .format("com.databricks.spark.redshift")
  22. .option("url",jdbcURL )
  23. .option("tempdir", tempS3Dir)
  24. .option("dbtable", "table_name")
  25. .option("forward_spark_s3_credentials","true")
  26. .load()
  27.  
  28. eventsDF.show()
  29.  
  30. Exception in thread "main" java.lang.NoClassDefFoundError: com/amazonaws/services/s3/model/S3ObjectInputStream
  31. at com.databricks.spark.redshift.DefaultSource.createRelation(DefaultSource.scala:51)
  32. at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:330)
  33. at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:152)
  34. at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:125)
  35. at new_test$.main(new_test.scala:42)
  36. at new_test.main(new_test.scala)
  37. Caused by: java.lang.ClassNotFoundException: com.amazonaws.services.s3.model.S3ObjectInputStream
  38. at java.net.URLClassLoader.findClass(Unknown Source)
  39. at java.lang.ClassLoader.loadClass(Unknown Source)
  40. at sun.misc.Launcher$AppClassLoader.loadClass(Unknown Source)
  41. at java.lang.ClassLoader.loadClass(Unknown Source)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement