SHARE
TWEET

Untitled

a guest Nov 22nd, 2015 225 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. // SD1 and SD2
  2. val text = sc.textFile("/1.txt")
  3. val hBV = text.map(a=>a.toDouble)
  4. val hbv1 p pairHBV.map(a=>(a._2, a.2)
  5. val pairHBV = hBV.zipWithIndex
  6.  
  7. import org.apache.spark.rdd.PairRDDFunctions
  8. val tmp1 = pairHBV.map(a => (a._2, a._1))
  9. val tmp2 = pairHBV.map(a => (a._2 + 1, a._1))
  10. val hbvBase = tmp1.join(tmp2)
  11. val hbvBase1 = hbvBase.map(a => a._2)
  12.  
  13.  
  14. val h = hbvBase1.map(a=>(1, a._2-a._1, (a._2- a._1) * (a._2-a._1))).reduce((a, b) => (a._1+b._1,a._2+b._2,a._3+b._3))
  15.  
  16. (h._2-h._3)*(h._2-h._3) / h._1
  17. // mapPartition example 0
  18. val parallel = sc.parallelize(1 to 9, 3)
  19. parallel.mapPartitions( x => List(x.next).iterator).collect
  20. // res383: Array[Int] = Array(1, 4, 7)
  21. // compare to the same, but with default parallelize
  22. val parallel = sc.parallelize(1 to 9)
  23. parallel.mapPartitions( x => List(x.next).iterator).collect
  24. // res384: Array[Int] = Array(1, 2, 3, 4, 5, 6, 7, 8)
  25.  
  26. // mapPartition example 1
  27. val a = sc.parallelize(1 to 9, 3)
  28. def myfunc[T](iter: Iterator[T]) : Iterator[(T, T)] = {
  29.   var res = List[(T, T)]()
  30.   var pre = iter.next
  31.   while (iter.hasNext)
  32.   {
  33.     val cur = iter.next;
  34.     res .::= (pre, cur)
  35.     pre = cur;
  36.   }
  37.   res.iterator
  38. }
  39. a.mapPartitions(myfunc).collect
  40. //res0: Array[(Int, Int)] = Array((2,3), (1,2), (5,6), (4,5), (8,9), (7,8))
  41.  
  42. // mapPartition example 2
  43. val z = sc.parallelize(List("a","b","c","d","e","f"),2)
  44.  
  45. //lets first print out the contents of the RDD with partition labels
  46. def myfunc(index: Int, iter: Iterator[(String)]) : Iterator[String] = {
  47.   iter.toList.map(x => "[partID:" +  index + ", val: " + x + "]").iterator
  48. }
  49.  
  50. z.mapPartitionsWithIndex(myfunc).collect
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!
 
Top