Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def createDataset(ratings: RDD[Tuple2[Long,Rating]]): List[RDD[Rating]] = {
- val training = ratings.filter(x => x._1 < 6).values.repartition(numPartitions).persist
- val validation = ratings.filter(x => x._1 >= 6 && x._1 < 8).values.repartition(numPartitions).persist
- val test = ratings.filter(x => x._1 >= 8).values.persist
- val numTraining = training.count
- val numValidation = validation.count
- val numTest = test.count
- println(" Number Of Training ::: " + numTraining + " numValidation ::: " + numValidation + " ::: " + numTest)
- List(training,validation,test)
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement