• API
• FAQ
• Tools
• Archive
SHARE
TWEET Untitled a guest Nov 22nd, 2015 225 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
1. // SD1 and SD2
2. val text = sc.textFile("/1.txt")
3. val hBV = text.map(a=>a.toDouble)
4. val hbv1 p pairHBV.map(a=>(a._2, a.2)
5. val pairHBV = hBV.zipWithIndex
6.
7. import org.apache.spark.rdd.PairRDDFunctions
8. val tmp1 = pairHBV.map(a => (a._2, a._1))
9. val tmp2 = pairHBV.map(a => (a._2 + 1, a._1))
10. val hbvBase = tmp1.join(tmp2)
11. val hbvBase1 = hbvBase.map(a => a._2)
12.
13.
14. val h = hbvBase1.map(a=>(1, a._2-a._1, (a._2- a._1) * (a._2-a._1))).reduce((a, b) => (a._1+b._1,a._2+b._2,a._3+b._3))
15.
16. (h._2-h._3)*(h._2-h._3) / h._1
17. // mapPartition example 0
18. val parallel = sc.parallelize(1 to 9, 3)
19. parallel.mapPartitions( x => List(x.next).iterator).collect
20. // res383: Array[Int] = Array(1, 4, 7)
21. // compare to the same, but with default parallelize
22. val parallel = sc.parallelize(1 to 9)
23. parallel.mapPartitions( x => List(x.next).iterator).collect
24. // res384: Array[Int] = Array(1, 2, 3, 4, 5, 6, 7, 8)
25.
26. // mapPartition example 1
27. val a = sc.parallelize(1 to 9, 3)
28. def myfunc[T](iter: Iterator[T]) : Iterator[(T, T)] = {
29.   var res = List[(T, T)]()
30.   var pre = iter.next
31.   while (iter.hasNext)
32.   {
33.     val cur = iter.next;
34.     res .::= (pre, cur)
35.     pre = cur;
36.   }
37.   res.iterator
38. }
39. a.mapPartitions(myfunc).collect
40. //res0: Array[(Int, Int)] = Array((2,3), (1,2), (5,6), (4,5), (8,9), (7,8))
41.
42. // mapPartition example 2
43. val z = sc.parallelize(List("a","b","c","d","e","f"),2)
44.
45. //lets first print out the contents of the RDD with partition labels
46. def myfunc(index: Int, iter: Iterator[(String)]) : Iterator[String] = {
47.   iter.toList.map(x => "[partID:" +  index + ", val: " + x + "]").iterator
48. }
49.
50. z.mapPartitionsWithIndex(myfunc).collect
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!

Top