Advertisement
Guest User

Untitled

a guest
Jun 25th, 2019
66
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.80 KB | None | 0 0
  1. def registerKryoClasses(classes: Array[Class[_]]): SparkConf = { .. }
  2.  
  3. @transient lazy val conf: RuntimeConfig = new RuntimeConfig(sessionState.conf)
  4.  
  5. /**
  6. * Runtime configuration interface for Spark. To access this, use `SparkSession.conf`.
  7. *
  8. * Options set here are automatically propagated to the Hadoop configuration during I/O.
  9. *
  10. * @since 2.0.0
  11. */
  12. @InterfaceStability.Stable
  13. class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
  14.  
  15. val mysparkConf = SparkConf.set(someKey,someVal)
  16. SparkSession.builder.conf(mySparkConf)
  17.  
  18. sparkConf.registerKryoClasses(Array(classOf[org.apache.spark.sql.Row]))
  19.  
  20. scala> spark.conf.registerKryoClasses(Array(classOf[Row]))
  21.  
  22. implicit val generalRowEncoder: Encoder[Row] = org.apache.spark.sql.Encoders.kryo[Row]
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement