Advertisement
Guest User

Untitled

a guest
Oct 6th, 2017
77
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.13 KB | None | 0 0
  1. val query = "(select * from destinationlarge) as dest"
  2. val options = Map(
  3. "url" -> "jdbc:mysql://172.25.36.151:3306/test?useSSL=false",
  4. "driver" -> "com.mysql.jdbc.Driver",
  5. "dbtable" -> query,
  6. "user" -> "root",
  7. "password" -> "root")
  8.  
  9. val destination = spark.read.options(options).jdbc(options("url"), options("dbtable"), "0", 1, 5, 4, new java.util.Properties()).rdd.map(_.mkString(",")).persist(StorageLevel.DISK_ONLY)
  10.  
  11. Exception in thread "main" org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 6, datanode5, executor 6): ExecutorLostFailure (executor 6 exited caused by one of the running tasks) Reason: Executor heartbeat timed out after 139401 ms
  12. Driver stacktrace:
  13. at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)
  14. at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)
  15. at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)
  16. at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
  17. at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
  18. at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)
  19. at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
  20. at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
  21. at scala.Option.foreach(Option.scala:257)
  22. at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)
  23. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)
  24. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)
  25. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)
  26. at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
  27. at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)
  28. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1925)
  29. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1938)
  30. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1951)
  31. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1965)
  32. at org.apache.spark.rdd.RDD.count(RDD.scala:1158)
  33. at com.syntel.spark.sparkDVT$.main(sparkDVT.scala:68)
  34. at com.syntel.spark.sparkDVT.main(sparkDVT.scala)
  35. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  36. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  37. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  38. at java.lang.reflect.Method.invoke(Method.java:497)
  39. at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:750)
  40. at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187)
  41. at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212)
  42. at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126)
  43. at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement