Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- test("save - append - ArrayType.containsNull") {
- withTempPath { file =>
- val df = Seq.empty[Tuple1[Seq[Int]]].toDF("arrayVal")
- val nonNullSchema = StructType(df.schema.map {
- case f @ StructField(_, a: ArrayType, _, _) =>
- f.copy(dataType = a.copy(containsNull = false))
- case f => f
- })
- sqlContext.createDataFrame(df.rdd, nonNullSchema).save(file.getCanonicalPath)
- (Tuple1(Seq(1, 2)) :: Tuple1(null.asInstanceOf[Seq[Int]]) :: Nil)
- .toDF("arrayVal")
- .save(file.getCanonicalPath, SaveMode.Append)
- checkAnswer(
- parquetFile(file.getCanonicalPath),
- Row(ArrayBuffer(1, 2)) :: Row(null) :: Nil)
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement