Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- //funciones auxiliares
- def schemaCleaner(x: StructField): StructField = {
- if (x.dataType.isInstanceOf[DecimalType])
- StructField(x.name, DoubleType, nullable = x.nullable)
- else
- x
- }
- def castColumn(df: DataFrame, colName: String, newDataType: DataType): DataFrame = {
- df.withColumn(colName, df.col(colName).cast(newDataType))
- }
- //meollo
- val cleanSchema = df_load.schema.fields.map( x => schemaCleaner(x))
- cleanSchema.foreach(x=> println(x.toString()))
- val schemaDifferences = df_load.schema.zip(cleanSchema).filter(x=> if(x._1.dataType!=x._2.dataType) true else false).map(x=> (x._1.name,x._2.dataType))
- schemaDifferences.foreach(x=> println(s"${x._1} ${x._2.toString}"))
- val df_result = schemaDifferences.foldLeft(df_load){
- case (acc,value) => castColumn(acc,value._1,value._2)
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement