Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- libraryDependencies += "com.fasterxml.jackson.module" % "jackson-module-scala_2.11" % "2.8.4"
- libraryDependencies += "com.amazonaws" % "aws-java-sdk-core" % "1.11.79"
- libraryDependencies += "com.amazonaws" % "aws-java-sdk-s3" % "1.11.79"
- libraryDependencies += "org.apache.avro" % "avro-mapred" % "1.8.1"
- libraryDependencies += "com.amazonaws" % "aws-java-sdk-redshift" % "1.11.78"
- libraryDependencies += "com.databricks" % "spark-redshift_2.11" % "3.0.0-preview1"
- libraryDependencies += "org.alluxio" % "alluxio-core-client" % "1.3.0"
- libraryDependencies += "com.taxis99" %% "awsscala" % "0.7.3"
- libraryDependencies += "org.apache.hadoop" % "hadoop-aws" % "2.7.3"
- libraryDependencies += "org.apache.spark" %% "spark-core" % sparkVersion
- libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion
- libraryDependencies += "org.apache.spark" %% "spark-mllib" % sparkVersion
- val df = spark.read.jdbc(url_read,"public.test", prop).as[Schema.Message.Raw]
- .filter("message != ''")
- .filter("from_id >= 0")
- .limit(100)
- df.write
- .format("com.databricks.spark.redshift")
- .option("url", "jdbc:redshift://test.XXX.redshift.amazonaws.com:5439/test?user=test&password=testXXXXX")
- .option("dbtable", "table_test")
- .option("tempdir", "s3a://redshift_logs/")
- .option("forward_spark_s3_credentials", "true")
- .option("tempformat", "CSV")
- .option("jdbcdriver", "com.amazon.redshift.jdbc42.Driver")
- .mode(SaveMode.Overwrite)
- .save()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement