API tools faq
paste
Guest User

Untitled

a guest
Jun 1st, 2018
112
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 24.51 KB | None | 0 0
raw download clone embed print report
  1. val fp = "/user/user_id.txt"
  2. sc.textFile(fp).map { s =>
  3. val Array(did, info_s) = s.split("t")
  4. val info = info_s.split(",").map { kv =>
  5. val Array(k, v) = kv.split(":")
  6. (k, v.toDouble)
  7. }.toSeq
  8. (did, info)
  9. }
  10.  
  11. scala.MatchError: [Ljava.lang.String;@51443799 (of class [Ljava.lang.String;)
  12. at com.test.news.IO$$anonfun$1.apply(App.scala:58)
  13. at com.test.news.IO$$anonfun$1.apply(App.scala:57)
  14. at scala.collection.Iterator$$anon$11.next(Iterator.scala:409)
  15. at scala.collection.Iterator$class.foreach(Iterator.scala:893)
  16. at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
  17. at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59)
  18. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104)
  19. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48)
  20. at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:310)
  21. at scala.collection.AbstractIterator.to(Iterator.scala:1336)
  22. at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:302)
  23. at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1336)
  24. at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:289)
  25. at scala.collection.AbstractIterator.toArray(Iterator.scala:1336)
  26. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:912)
  27. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$13.apply(RDD.scala:912)
  28. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1916)
  29. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1916)
  30. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70)
  31. at org.apache.spark.scheduler.Task.run(Task.scala:86)
  32. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
  33. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  34. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  35. at java.lang.Thread.run(Thread.java:745)
  36.  
  37. val (id, info) = ("123", "word:123")
  38.  
  39. sc.textFile("user_id.txt").map{ line =>
  40. val fields = line.split("t")
  41. val info = fields(1).split(",").map { kv =>
  42. val pairs = kv.split(":")
  43. (pairs(0), pairs(1).toDouble)
  44. }.toSeq
  45. (fields(0), info)
  46. }.collect()
  47.  
  48. # Array[(String, Seq[(String, Double)])] = Array((12334,WrappedArray((this,23.0), (word,21.0), (teacher,23.0))))
  49.  
  50. val Array(k, v, _*) = Array(1, 2, 3, 4, 5)
  51. #k: Int = 1
  52. #v: Int = 2
  53.  
  54. sc.textFile("user_id.txt").map{ line =>
  55. val Array(id, info_s, _*) = line.split("t")
  56. val info = info_s.split(",").map { kv =>
  57. val Array(key, value, _*) = kv.split(":")
  58. (key, value.toDouble)
  59. }.toSeq
  60. (id, info)
  61. }.collect()
  62.  
  63. # Array[(String, Seq[(String, Double)])] = Array((12334,WrappedArray((this,23.0), (word,21.0), (teacher,23.0))))
  64.  
  65. scala> val Array(k, v) = "1,2".split(",")
  66. k: String = 1
  67. v: String = 2
  68.  
  69. scala> val Array(k, v) = "1,2,3".split(",")
  70. scala.MatchError: [Ljava.lang.String;@508dec2b (of class [Ljava.lang.String;)
  71.  
  72. val (k, v) = kv.split(":") match {
  73. case Array(f1, f2) => (f1, f2)
  74. case Array(elems) => fatal("found invalid K/V pair: expected 2 elements, found ${elems.length}")
  75. }
  76.  
  77. I got an error for below code (recommendation): below
  78.  
  79. import java.util.Properties
  80.  
  81. import org.apache.spark.broadcast.Broadcast
  82. import org.apache.spark.mllib.recommendation.{ALS, MatrixFactorizationModel, Rating}
  83. import org.apache.spark.rdd.RDD
  84. import org.apache.spark.sql.types.{StringType, StructField, StructType}
  85. import org.apache.spark.sql.{Row, SQLContext}
  86. import org.apache.spark.storage.StorageLevel
  87. import org.apache.spark.{SparkConf, SparkContext}
  88.  
  89. import scala.collection.Map
  90.  
  91. case class MovieRatings(userID: String, movieID: Int, rating: Double) extends scala.Serializable
  92.  
  93. case class Movies(MovieID: String, MovieName: String) extends scala.Serializable
  94.  
  95. object RecommendMovie {
  96. var sqlContext: SQLContext = _
  97. var BMovieAndName: Broadcast[Map[String, String]] = _
  98. var userIDToInt: RDD[(String, Long)] = _
  99. var model: MatrixFactorizationModel = _
  100.  
  101. def main1(args: Array[String]): Unit = {
  102. /* val conf = new SparkConf().setAppName("RecommendMovies")
  103. val sc = new SparkContext(conf)
  104. sqlContext = new SQLContext(sc) */
  105. val HotMovies = sc.textFile("/home/cloudera/machine_learning/hot_movies.csv")
  106. val UserMovies = sc.textFile("/home/cloudera/machine_learning/user_movies.csv")
  107. MovieModel(sc, HotMovies, UserMovies)
  108. val username = if (args.length > 0) args(0) else "wangymm"
  109. val userIDMap: Map[String, Int] = userIDToInt.collectAsMap().map { case (s, l) => (s, l.toInt) }
  110. RecommendByName(username, UserMovies, BMovieAndName, userIDMap, model)
  111. sc.stop()
  112. }
  113.  
  114. def BuildMovies(HotMovies: RDD[String]): RDD[Movies] = {
  115. HotMovies.map { line => val Array(movieID, pref, movieName) = line.split(',')
  116. if (movieID.isEmpty) {
  117. null
  118. }
  119. else {
  120. Movies(movieID, movieName)
  121. }
  122. }
  123. }
  124. def BuildRating(UserMovies: RDD[String]): RDD[MovieRatings] = {
  125. UserMovies.map {
  126. line => val Array(userID, moviesID, countStr) = line.split(',').map(_.trim)
  127. var count = countStr.toInt
  128. count = if (count == -1) 3 else count
  129. MovieRatings(userID, moviesID.toInt, count)
  130. }
  131. }
  132. def MovieModel(sc: SparkContext, HotMovies: RDD[String], UserMovies: RDD[String]): Unit = {
  133. val MovieAndName = BuildMovies(HotMovies)
  134. BMovieAndName = sc.broadcast(MovieAndName.map { movie => (movie.MovieID, movie.MovieName) }.collectAsMap())
  135. val MovieRatingResult = BuildRating(UserMovies)
  136. userIDToInt = MovieRatingResult.map(_.userID).distinct().zipWithUniqueId()
  137. val reverseUserID: RDD[(Int, String)] = userIDToInt.map { case (s, l) => (l.toInt, s) }
  138. val userIDMap: Map[String, Int] = userIDToInt.collectAsMap().map { case (s, l) => (s, l.toInt) }
  139. val BUserIDMap = sc.broadcast(userIDMap)
  140. val BreverseUserID = sc.broadcast(reverseUserID.collectAsMap())
  141. val rating: RDD[Rating] = MovieRatingResult.map { line => Rating(BUserIDMap.value.get(line.userID).get, line.movieID, line.rating) }.persist
  142.  
  143. (StorageLevel.MEMORY_AND_DISK)
  144. model = ALS.train(rating, 50, 10, 0.0001)
  145. rating.unpersist()
  146. val allRecommends = model.recommendProductsForUsers(5).map {
  147. case (userid, recommends) => {
  148. var recommendStr = ""
  149. for (r <- recommends) {
  150. recommendStr += r.product + ":" + BMovieAndName.value.getOrElse(r.product.toString, "") + ","
  151. }
  152. if (recommendStr.endsWith(",")) {
  153. recommendStr = recommendStr.substring(0, recommendStr.length - 1)
  154. }
  155. (BreverseUserID.value.get(userid).get, recommendStr)
  156. }
  157. }
  158. allRecommends.saveAsTextFile("/home/cloudera/machine_learning/result.csv")
  159. HistoryMovies(sc, HotMovies, UserMovies)
  160.  
  161. model.userFeatures.unpersist()
  162. model.productFeatures.unpersist()
  163. val resultdata = sc.textFile("/home/cloudera/machine_learning/result.csv").map(_.split(","))
  164. val schema = StructType(
  165. List(
  166. StructField("userID", StringType, false),
  167. StructField("movie01", StringType, false),
  168. StructField("movie02", StringType, false),
  169. StructField("movie03", StringType, false),
  170. StructField("movie04", StringType, false),
  171. StructField("movie05", StringType, false)
  172. )
  173. )
  174. val rows = resultdata.map(line => Row(line(0).substring(1).toString().trim, line(1).toString.trim, line(2).toString.trim, line(3).toString.trim, line
  175.  
  176. (4).toString.trim, line(5).toString.trim))
  177. val dataRDD = sqlContext.createDataFrame(rows, schema)
  178. val prop = new Properties()
  179. prop.put("user", "root")
  180. prop.put("password", "cloudera")
  181. dataRDD.write.mode("overwrite").jdbc("jdbc:mysql://localhost:3306/retail_db?characterEncoding=utf8", "retail_db", prop)
  182. }
  183. def RecommendByName(UserName: String, UserMovies: RDD[String], BMovieAndName: Broadcast[Map[String, String]], userIDMap: Map[String, Int], model:
  184.  
  185. MatrixFactorizationModel): Unit = {
  186. val UserID = userIDMap.filter {
  187. case (username, userid) => username == UserName
  188. }.values.head
  189. val recommendations = model.recommendProducts(UserID, 5)
  190. val RecommendMovieResult = recommendations.map(_.product).toSet
  191. val UserSeen = UserMovies.map(_.split(",")).filter { case Array(user, _, _) => user.trim == UserName }
  192. val MovieSeen = UserSeen.map { case Array(_, movieid, _) => movieid.toInt }.collect().toSet
  193. }
  194. def HistoryMovies(sc: SparkContext, HotMovies: RDD[String], UserMovies: RDD[String]): Unit = {
  195. val UserHaveSeen: RDD[(String, String)] = UserMovies.map(_.split(",")).map { case Array(userid, movieid, _) => (userid.toString, movieid.toString) }
  196. val UserSeen = UserHaveSeen.reduceByKey((x, y) => x + "," + y)
  197. val reverse = UserHaveSeen.map { case (u, m) => (m, u) }
  198. val UserAndName: RDD[(String, String)] = HotMovies.map(_.split(",")).map { case Array(movieid, pref, moviename) => (movieid.toString, moviename.toString) }
  199. val resultRDD = reverse.join(UserAndName).map(_._2).reduceByKey((x, y) => x + "," + y)
  200. resultRDD.saveAsTextFile("/home/cloudera/machine_learning/history.csv")
  201. val schema = StructType(
  202. List(
  203. StructField("userid", StringType, false),
  204. StructField("historymovies", StringType, false)
  205. )
  206. )
  207. val rows = resultRDD.map { case (key, values) => Row(key.toString.trim, values.toString.trim) }
  208. val dataRDD = sqlContext.createDataFrame(rows, schema)
  209. val prop = new Properties()
  210. prop.put("user", "root")
  211. prop.put("password", "cloudera")
  212. dataRDD.write.mode("overwrite").jdbc("jdbc:mysql://localhost:3306/retail_db?characterEncoding=utf8", "history1", prop)
  213.  
  214. }
  215. }
  216.  
  217. Got an error:
  218.  
  219. 18/06/01 07:37:10 ERROR executor.Executor: Exception in task 0.0 in stage 3.0 (TID 3)
  220. scala.MatchError: [Ljava.lang.String;@2429eb1 (of class [Ljava.lang.String;)
  221. at $line126.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$$anonfun$BuildMovies$1.apply(<console>:123)
  222. at $line126.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$$anonfun$BuildMovies$1.apply(<console>:123)
  223. at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
  224. at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
  225. at scala.collection.Iterator$class.foreach(Iterator.scala:727)
  226. at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
  227. at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)
  228. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)
  229. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)
  230. at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)
  231. at scala.collection.AbstractIterator.to(Iterator.scala:1157)
  232. at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)
  233. at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157)
  234. at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)
  235. at scala.collection.AbstractIterator.toArray(Iterator.scala:1157)
  236. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:927)
  237. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:927)
  238. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
  239. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
  240. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
  241. at org.apache.spark.scheduler.Task.run(Task.scala:89)
  242. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:242)
  243. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  244. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  245. at java.lang.Thread.run(Thread.java:745)
  246. 18/06/01 07:37:10 WARN scheduler.TaskSetManager: Lost task 0.0 in stage 3.0 (TID 3, localhost, executor driver): scala.MatchError: [Ljava.lang.String;@2429eb1 (of class [Ljava.lang.String;)
  247. at $line126.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$$anonfun$BuildMovies$1.apply(<console>:123)
  248. at $line126.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$$anonfun$BuildMovies$1.apply(<console>:123)
  249. at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
  250. at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
  251. at scala.collection.Iterator$class.foreach(Iterator.scala:727)
  252. at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
  253. at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)
  254. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)
  255. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)
  256. at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)
  257. at scala.collection.AbstractIterator.to(Iterator.scala:1157)
  258. at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)
  259. at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157)
  260. at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)
  261. at scala.collection.AbstractIterator.toArray(Iterator.scala:1157)
  262. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:927)
  263. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:927)
  264. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
  265. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
  266. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
  267. at org.apache.spark.scheduler.Task.run(Task.scala:89)
  268. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:242)
  269. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  270. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  271. at java.lang.Thread.run(Thread.java:745)
  272.  
  273. 18/06/01 07:37:10 ERROR scheduler.TaskSetManager: Task 0 in stage 3.0 failed 1 times; aborting job
  274. org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 3.0 failed 1 times, most recent failure: Lost task 0.0 in stage 3.0 (TID 3, localhost, executor driver): scala.MatchError: [Ljava.lang.String;@2429eb1 (of class [Ljava.lang.String;)
  275. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$$anonfun$BuildMovies$1.apply(<console>:123)
  276. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$$anonfun$BuildMovies$1.apply(<console>:123)
  277. at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
  278. at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
  279. at scala.collection.Iterator$class.foreach(Iterator.scala:727)
  280. at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
  281. at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)
  282. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)
  283. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)
  284. at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)
  285. at scala.collection.AbstractIterator.to(Iterator.scala:1157)
  286. at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)
  287. at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157)
  288. at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)
  289. at scala.collection.AbstractIterator.toArray(Iterator.scala:1157)
  290. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:927)
  291. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:927)
  292. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
  293. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
  294. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
  295. at org.apache.spark.scheduler.Task.run(Task.scala:89)
  296. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:242)
  297. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  298. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  299. at java.lang.Thread.run(Thread.java:745)
  300.  
  301. Driver stacktrace:
  302. at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1457)
  303. at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1445)
  304. at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1444)
  305. at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
  306. at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
  307. at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1444)
  308. at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)
  309. at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:799)
  310. at scala.Option.foreach(Option.scala:236)
  311. at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799)
  312. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1668)
  313. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1627)
  314. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1616)
  315. at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
  316. at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620)
  317. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1862)
  318. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1875)
  319. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1888)
  320. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1959)
  321. at org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:927)
  322. at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
  323. at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
  324. at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
  325. at org.apache.spark.rdd.RDD.collect(RDD.scala:926)
  326. at org.apache.spark.rdd.PairRDDFunctions$$anonfun$collectAsMap$1.apply(PairRDDFunctions.scala:741)
  327. at org.apache.spark.rdd.PairRDDFunctions$$anonfun$collectAsMap$1.apply(PairRDDFunctions.scala:740)
  328. at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
  329. at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:111)
  330. at org.apache.spark.rdd.RDD.withScope(RDD.scala:316)
  331. at org.apache.spark.rdd.PairRDDFunctions.collectAsMap(PairRDDFunctions.scala:740)
  332. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$.MovieModel(<console>:142)
  333. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$.main1(<console>:115)
  334. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:106)
  335. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:111)
  336. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:113)
  337. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:115)
  338. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:117)
  339. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:119)
  340. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:121)
  341. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:123)
  342. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:125)
  343. at $iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:127)
  344. at $iwC$$iwC$$iwC$$iwC.<init>(<console>:129)
  345. at $iwC$$iwC$$iwC.<init>(<console>:131)
  346. at $iwC$$iwC.<init>(<console>:133)
  347. at $iwC.<init>(<console>:135)
  348. at <init>(<console>:137)
  349. at .<init>(<console>:141)
  350. at .<clinit>(<console>)
  351. at .<init>(<console>:7)
  352. at .<clinit>(<console>)
  353. at $print(<console>)
  354. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  355. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  356. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  357. at java.lang.reflect.Method.invoke(Method.java:606)
  358. at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1045)
  359. at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1326)
  360. at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:821)
  361. at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:852)
  362. at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:800)
  363. at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
  364. at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
  365. at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
  366. at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:657)
  367. at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:665)
  368. at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$loop(SparkILoop.scala:670)
  369. at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:997)
  370. at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
  371. at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
  372. at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
  373. at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
  374. at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1064)
  375. at org.apache.spark.repl.Main$.main(Main.scala:35)
  376. at org.apache.spark.repl.Main.main(Main.scala)
  377. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  378. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  379. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  380. at java.lang.reflect.Method.invoke(Method.java:606)
  381. at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:730)
  382. at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
  383. at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
  384. at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
  385. at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
  386. Caused by: scala.MatchError: [Ljava.lang.String;@2429eb1 (of class [Ljava.lang.String;)
  387. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$$anonfun$BuildMovies$1.apply(<console>:123)
  388. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$RecommendMovie$$anonfun$BuildMovies$1.apply(<console>:123)
  389. at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
  390. at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
  391. at scala.collection.Iterator$class.foreach(Iterator.scala:727)
  392. at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
  393. at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)
  394. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)
  395. at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)
  396. at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)
  397. at scala.collection.AbstractIterator.to(Iterator.scala:1157)
  398. at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)
  399. at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157)
  400. at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)
  401. at scala.collection.AbstractIterator.toArray(Iterator.scala:1157)
  402. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:927)
  403. at org.apache.spark.rdd.RDD$$anonfun$collect$1$$anonfun$12.apply(RDD.scala:927)
  404. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
  405. at org.apache.spark.SparkContext$$anonfun$runJob$5.apply(SparkContext.scala:1888)
  406. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)
  407. at org.apache.spark.scheduler.Task.run(Task.scala:89)
  408. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:242)
  409. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
  410. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
  411. at java.lang.Thread.run(Thread.java:745)
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!