Advertisement
Guest User

Untitled

a guest
Jul 28th, 2014
205
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Scala 0.59 KB | None | 0 0
  1. def createDataset(ratings: RDD[Tuple2[Long,Rating]]): List[RDD[Rating]] = {
  2.  
  3.     val training = ratings.filter(x => x._1 < 6).values.repartition(numPartitions).persist
  4.     val validation = ratings.filter(x => x._1 >= 6 && x._1 < 8).values.repartition(numPartitions).persist
  5.     val test = ratings.filter(x => x._1 >= 8).values.persist
  6.     val numTraining = training.count
  7.     val numValidation = validation.count
  8.     val numTest = test.count
  9.  
  10.     println(" Number Of Training ::: " + numTraining + " numValidation ::: " + numValidation + " ::: " + numTest)
  11.     List(training,validation,test)
  12.   }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement