Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- val sparkVersion = "2.4.3"
- val jacksonVersion = "2.8.7"
- dependencyOverrides ++= Seq(
- "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion,
- "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
- "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonVersion
- )
- libraryDependencies ++= Seq(
- "org.apache.spark" %% "spark-core" % sparkVersion % "provided" ,
- "org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
- "org.apache.spark" %% "spark-hive" % sparkVersion % "provided",
- "org.apache.spark" %% "spark-avro" % "2.4.3",
- "io.confluent" % "kafka-avro-serializer" % "5.0.1",
- "org.apache.avro" % "avro" % "1.8.2"
- )
- assemblyMergeStrategy in assembly := {
- case PathList("META-INF", xs@_*) => MergeStrategy.discard
- case x => MergeStrategy.first
- }
- Seq(1, 2, 3).toDF("id")
- .write
- .format("avro")
- .mode(SaveMode.Overwrite)
- .save("testavro")
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement