SHARE
TWEET

Untitled

a guest May 21st, 2016 174 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import org.apache.spark.SparkContext
  2. import org.apache.spark.SparkConf
  3.  
  4. import org.apache.log4j.Logger
  5. import org.apache.log4j.Level
  6. import org.apache.spark.rdd.RDD
  7.  
  8. import scala.tools.nsc.matching.Matrix
  9. import scala.util.control.Breaks
  10.  
  11. /**
  12.   * Created by Alexey on 25.04.2016.
  13.   */
  14. object MultMatrix {
  15.   def main(args: Array[String]) {
  16.  
  17.     Logger.getLogger("org").setLevel(Level.OFF)
  18.     Logger.getLogger("akka").setLevel(Level.OFF)
  19.  
  20.     if (args.length < 1) {
  21.       System.err.println("Usage: SparkGrep <host>")
  22.       System.exit(1)
  23.     }
  24.     val conf = new SparkConf().setAppName("SparkGrep").setMaster(args(0))
  25.     val sc = new SparkContext(conf)
  26.  
  27.     val m2: List[List[Int]] =
  28.       List(
  29.         List(1, 2, 3, 1),
  30.         List(2, 1, 2, 1),
  31.         List(1, 2, 1, 1))
  32.  
  33.     val m1: List[List[Int]] =
  34.       List(
  35.         List(5, 2, 3),
  36.         List(2, 6, 2),
  37.         List(7, 2, 8),
  38.         List(1, 2, 8)
  39.       )
  40.  
  41.     val m2Prepared = sc.parallelize(0.to(m2.length - 1).map(i => 0.to(m2(0).length - 1).map(j => (i, j) -> m2(i)(j))).flatMap(q => q))
  42.     val m1Prepared = sc.parallelize(0.to(m1.length - 1).map(i => 0.to(m1(0).length - 1).map(j => (i, j) -> m1(i)(j))).flatMap(q => q))
  43.  
  44.     val rows = m1Prepared.groupBy(q => q._1._1).sortBy(q => q._1).map(q => q._1 -> q._2.map(q => q._2))
  45.     val columns = m2Prepared.groupBy(q => q._1._2).sortBy(q => q._1).map(q => q._1 -> q._2.map(q => q._2))
  46.  
  47.     val t1 = rows.collect()
  48.     val t2 = columns.collect()
  49.  
  50.     val cart = rows.cartesian(columns)
  51.     //val maetrixWithInd = 0.to(matrix.length)
  52.  
  53.     val x = sc.parallelize(List("spark rdd example", "sample example"), 2)
  54.     val y = x.flatMap(x => x.split(" "))
  55.  
  56.     val r = cart.map(q => (q._1._1, q._2._1) -> q._1._2.zip(q._2._2).map(w => w._1 * w._2).reduce((q, w) => q + w))
  57.  
  58.     val result = r.collect().toMap
  59.  
  60.     //val result = zipped.map(q=>q._1+q._2).reduce()
  61.  
  62.     val resultRowsCount = m1.length
  63.     val resultColumnsCount = m2(0).length
  64.  
  65.     for(i <- 0 to resultColumnsCount*resultRowsCount-1)
  66.           {
  67.             val col = i%resultColumnsCount
  68.             val row = i/resultColumnsCount
  69.  
  70.  
  71.  
  72.             print(result.get(row,col).get + "\t")
  73.  
  74.             if (col==resultColumnsCount-1)
  75.               println()
  76.           }
  77.  
  78.  
  79.   }
  80. }
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top