Advertisement
Guest User

Untitled

a guest
Apr 18th, 2015
189
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.68 KB | None | 0 0
  1. test("save - append - ArrayType.containsNull") {
  2. withTempPath { file =>
  3. val df = Seq.empty[Tuple1[Seq[Int]]].toDF("arrayVal")
  4. val nonNullSchema = StructType(df.schema.map {
  5. case f @ StructField(_, a: ArrayType, _, _) =>
  6. f.copy(dataType = a.copy(containsNull = false))
  7. case f => f
  8. })
  9.  
  10. sqlContext.createDataFrame(df.rdd, nonNullSchema).save(file.getCanonicalPath)
  11.  
  12. (Tuple1(Seq(1, 2)) :: Tuple1(null.asInstanceOf[Seq[Int]]) :: Nil)
  13. .toDF("arrayVal")
  14. .save(file.getCanonicalPath, SaveMode.Append)
  15.  
  16. checkAnswer(
  17. parquetFile(file.getCanonicalPath),
  18. Row(ArrayBuffer(1, 2)) :: Row(null) :: Nil)
  19. }
  20. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement