Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def registerKryoClasses(classes: Array[Class[_]]): SparkConf = { .. }
- @transient lazy val conf: RuntimeConfig = new RuntimeConfig(sessionState.conf)
- /**
- * Runtime configuration interface for Spark. To access this, use `SparkSession.conf`.
- *
- * Options set here are automatically propagated to the Hadoop configuration during I/O.
- *
- * @since 2.0.0
- */
- @InterfaceStability.Stable
- class RuntimeConfig private[sql](sqlConf: SQLConf = new SQLConf) {
- val mysparkConf = SparkConf.set(someKey,someVal)
- SparkSession.builder.conf(mySparkConf)
- sparkConf.registerKryoClasses(Array(classOf[org.apache.spark.sql.Row]))
- scala> spark.conf.registerKryoClasses(Array(classOf[Row]))
- implicit val generalRowEncoder: Encoder[Row] = org.apache.spark.sql.Encoders.kryo[Row]
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement