Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- val pairRDD1 = sc.parallelize(List( ("cat",2), ("girl", 5), ("book", 4),("Tom", 12)))
- val pairRDD2 = sc.parallelize(List( ("cat",2), ("cup", 5), ("mouse", 4),("girl", 12)))
- val kk = pairRDD1.fullOuterJoin(pairRDD2).collect
- kk: Array[(String, (Option[Int], Option[Int]))] = Array((book,(Some(4),None)), (Tom,(Some(12),None)), (girl,(Some(5),Some(12))), (mouse,(None,Some(4))), (cup,(None,Some(5))), (cat,(Some(2),Some(2))))
- pairRDD1.fullOuterJoin(pairRDD2).mapValues(pair => (pair._1.getOrElse(0), pair._2.getOrElse(0)))
- kk.map { case (k, pair) => (k, (pair._1.getOrElse(0), pair._2.getOrElse(0))) }
- val ss = SparkSession.builder().master("local[*]").getOrCreate()
- val sc = ss.sparkContext
- import ss.implicits._
- val pairRDD1 = sc.parallelize(List(("cat", 2,9999), ("girl", 5,8888), ("book", 4,9999), ("Tom", 12,6666)))
- val pairRDD2 = sc.parallelize(List(("cat", 2,9999), ("cup", 5,7777), ("mouse", 4,3333), ("girl", 12,1111)))
- val df1 = pairRDD1.toDF
- val df2 = pairRDD2.toDF
- val joined = df1.join(df2, df1.col("_1") === df2.col("_1"),"fullouter")
- joined.show()
- +----+----+----+-----+----+----+
- | _1| _2| _3| _1| _2| _3|
- +----+----+----+-----+----+----+
- |girl| 5|8888| girl| 12|1111|
- | Tom| 12|6666| null|null|null|
- | cat| 2|9999| cat| 2|9999|
- |null|null|null| cup| 5|7777|
- |null|null|null|mouse| 4|3333|
- |book| 4|9999| null|null|null|
- +----+----+----+-----+----+----+
Add Comment
Please, Sign In to add comment