Advertisement
Siodh

cambia Schemas

Jan 3rd, 2019
60
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Scala 0.85 KB | None | 0 0
  1. //funciones auxiliares
  2. def schemaCleaner(x: StructField): StructField = {
  3.     if (x.dataType.isInstanceOf[DecimalType])
  4.       StructField(x.name, DoubleType, nullable = x.nullable)
  5.     else
  6.       x
  7.   }
  8.  
  9.   def castColumn(df: DataFrame, colName: String, newDataType: DataType): DataFrame = {
  10.     df.withColumn(colName, df.col(colName).cast(newDataType))
  11.   }
  12.  
  13.  //meollo
  14. val cleanSchema = df_load.schema.fields.map( x => schemaCleaner(x))
  15.       cleanSchema.foreach(x=> println(x.toString()))
  16.  
  17.       val schemaDifferences = df_load.schema.zip(cleanSchema).filter(x=> if(x._1.dataType!=x._2.dataType) true else false).map(x=> (x._1.name,x._2.dataType))
  18.       schemaDifferences.foreach(x=> println(s"${x._1} ${x._2.toString}"))
  19.  
  20.       val df_result = schemaDifferences.foldLeft(df_load){
  21.         case (acc,value) => castColumn(acc,value._1,value._2)
  22.       }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement